1 //===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file This pass tries to apply several peephole SDWA patterns. 11 /// 12 /// E.g. original: 13 /// V_LSHRREV_B32_e32 %0, 16, %1 14 /// V_ADD_I32_e32 %2, %0, %3 15 /// V_LSHLREV_B32_e32 %4, 16, %2 16 /// 17 /// Replace: 18 /// V_ADD_I32_sdwa %4, %1, %3 19 /// dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD 20 /// 21 //===----------------------------------------------------------------------===// 22 23 #include "AMDGPU.h" 24 #include "AMDGPUSubtarget.h" 25 #include "SIDefines.h" 26 #include "SIInstrInfo.h" 27 #include "SIRegisterInfo.h" 28 #include "Utils/AMDGPUBaseInfo.h" 29 #include "llvm/ADT/None.h" 30 #include "llvm/ADT/Optional.h" 31 #include "llvm/ADT/STLExtras.h" 32 #include "llvm/ADT/SmallVector.h" 33 #include "llvm/ADT/Statistic.h" 34 #include "llvm/CodeGen/MachineBasicBlock.h" 35 #include "llvm/CodeGen/MachineFunction.h" 36 #include "llvm/CodeGen/MachineFunctionPass.h" 37 #include "llvm/CodeGen/MachineInstr.h" 38 #include "llvm/CodeGen/MachineInstrBuilder.h" 39 #include "llvm/CodeGen/MachineOperand.h" 40 #include "llvm/CodeGen/MachineRegisterInfo.h" 41 #include "llvm/CodeGen/TargetRegisterInfo.h" 42 #include "llvm/MC/LaneBitmask.h" 43 #include "llvm/MC/MCInstrDesc.h" 44 #include "llvm/Pass.h" 45 #include "llvm/Support/Debug.h" 46 #include "llvm/Support/raw_ostream.h" 47 #include <algorithm> 48 #include <cassert> 49 #include <cstdint> 50 #include <memory> 51 #include <unordered_map> 52 53 using namespace llvm; 54 55 #define DEBUG_TYPE "si-peephole-sdwa" 56 57 STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found."); 58 STATISTIC(NumSDWAInstructionsPeepholed, 59 "Number of instruction converted to SDWA."); 60 61 namespace { 62 63 class SDWAOperand; 64 class SDWADstOperand; 65 66 class SIPeepholeSDWA : public MachineFunctionPass { 67 public: 68 using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>; 69 70 private: 71 MachineRegisterInfo *MRI; 72 const SIRegisterInfo *TRI; 73 const SIInstrInfo *TII; 74 75 std::unordered_map<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands; 76 std::unordered_map<MachineInstr *, SDWAOperandsVector> PotentialMatches; 77 SmallVector<MachineInstr *, 8> ConvertedInstructions; 78 79 Optional<int64_t> foldToImm(const MachineOperand &Op) const; 80 81 public: 82 static char ID; 83 84 SIPeepholeSDWA() : MachineFunctionPass(ID) { 85 initializeSIPeepholeSDWAPass(*PassRegistry::getPassRegistry()); 86 } 87 88 bool runOnMachineFunction(MachineFunction &MF) override; 89 void matchSDWAOperands(MachineFunction &MF); 90 std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI); 91 bool isConvertibleToSDWA(const MachineInstr &MI, const SISubtarget &ST) const; 92 bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands); 93 void legalizeScalarOperands(MachineInstr &MI, const SISubtarget &ST) const; 94 95 StringRef getPassName() const override { return "SI Peephole SDWA"; } 96 97 void getAnalysisUsage(AnalysisUsage &AU) const override { 98 AU.setPreservesCFG(); 99 MachineFunctionPass::getAnalysisUsage(AU); 100 } 101 }; 102 103 class SDWAOperand { 104 private: 105 MachineOperand *Target; // Operand that would be used in converted instruction 106 MachineOperand *Replaced; // Operand that would be replace by Target 107 108 public: 109 SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp) 110 : Target(TargetOp), Replaced(ReplacedOp) { 111 assert(Target->isReg()); 112 assert(Replaced->isReg()); 113 } 114 115 virtual ~SDWAOperand() = default; 116 117 virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII) = 0; 118 virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0; 119 120 MachineOperand *getTargetOperand() const { return Target; } 121 MachineOperand *getReplacedOperand() const { return Replaced; } 122 MachineInstr *getParentInst() const { return Target->getParent(); } 123 124 MachineRegisterInfo *getMRI() const { 125 return &getParentInst()->getParent()->getParent()->getRegInfo(); 126 } 127 128 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 129 virtual void print(raw_ostream& OS) const = 0; 130 void dump() const { print(dbgs()); } 131 #endif 132 }; 133 134 using namespace AMDGPU::SDWA; 135 136 class SDWASrcOperand : public SDWAOperand { 137 private: 138 SdwaSel SrcSel; 139 bool Abs; 140 bool Neg; 141 bool Sext; 142 143 public: 144 SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp, 145 SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false, 146 bool Sext_ = false) 147 : SDWAOperand(TargetOp, ReplacedOp), 148 SrcSel(SrcSel_), Abs(Abs_), Neg(Neg_), Sext(Sext_) {} 149 150 MachineInstr *potentialToConvert(const SIInstrInfo *TII) override; 151 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override; 152 153 SdwaSel getSrcSel() const { return SrcSel; } 154 bool getAbs() const { return Abs; } 155 bool getNeg() const { return Neg; } 156 bool getSext() const { return Sext; } 157 158 uint64_t getSrcMods(const SIInstrInfo *TII, 159 const MachineOperand *SrcOp) const; 160 161 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 162 void print(raw_ostream& OS) const override; 163 #endif 164 }; 165 166 class SDWADstOperand : public SDWAOperand { 167 private: 168 SdwaSel DstSel; 169 DstUnused DstUn; 170 171 public: 172 173 SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp, 174 SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD) 175 : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {} 176 177 MachineInstr *potentialToConvert(const SIInstrInfo *TII) override; 178 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override; 179 180 SdwaSel getDstSel() const { return DstSel; } 181 DstUnused getDstUnused() const { return DstUn; } 182 183 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 184 void print(raw_ostream& OS) const override; 185 #endif 186 }; 187 188 class SDWADstPreserveOperand : public SDWADstOperand { 189 private: 190 MachineOperand *Preserve; 191 192 public: 193 SDWADstPreserveOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp, 194 MachineOperand *PreserveOp, SdwaSel DstSel_ = DWORD) 195 : SDWADstOperand(TargetOp, ReplacedOp, DstSel_, UNUSED_PRESERVE), 196 Preserve(PreserveOp) {} 197 198 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override; 199 200 MachineOperand *getPreservedOperand() const { return Preserve; } 201 202 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 203 void print(raw_ostream& OS) const override; 204 #endif 205 }; 206 207 } // end anonymous namespace 208 209 INITIALIZE_PASS(SIPeepholeSDWA, DEBUG_TYPE, "SI Peephole SDWA", false, false) 210 211 char SIPeepholeSDWA::ID = 0; 212 213 char &llvm::SIPeepholeSDWAID = SIPeepholeSDWA::ID; 214 215 FunctionPass *llvm::createSIPeepholeSDWAPass() { 216 return new SIPeepholeSDWA(); 217 } 218 219 220 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 221 static raw_ostream& operator<<(raw_ostream &OS, const SdwaSel &Sel) { 222 switch(Sel) { 223 case BYTE_0: OS << "BYTE_0"; break; 224 case BYTE_1: OS << "BYTE_1"; break; 225 case BYTE_2: OS << "BYTE_2"; break; 226 case BYTE_3: OS << "BYTE_3"; break; 227 case WORD_0: OS << "WORD_0"; break; 228 case WORD_1: OS << "WORD_1"; break; 229 case DWORD: OS << "DWORD"; break; 230 } 231 return OS; 232 } 233 234 static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) { 235 switch(Un) { 236 case UNUSED_PAD: OS << "UNUSED_PAD"; break; 237 case UNUSED_SEXT: OS << "UNUSED_SEXT"; break; 238 case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break; 239 } 240 return OS; 241 } 242 243 static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) { 244 Operand.print(OS); 245 return OS; 246 } 247 248 LLVM_DUMP_METHOD 249 void SDWASrcOperand::print(raw_ostream& OS) const { 250 OS << "SDWA src: " << *getTargetOperand() 251 << " src_sel:" << getSrcSel() 252 << " abs:" << getAbs() << " neg:" << getNeg() 253 << " sext:" << getSext() << '\n'; 254 } 255 256 LLVM_DUMP_METHOD 257 void SDWADstOperand::print(raw_ostream& OS) const { 258 OS << "SDWA dst: " << *getTargetOperand() 259 << " dst_sel:" << getDstSel() 260 << " dst_unused:" << getDstUnused() << '\n'; 261 } 262 263 LLVM_DUMP_METHOD 264 void SDWADstPreserveOperand::print(raw_ostream& OS) const { 265 OS << "SDWA preserve dst: " << *getTargetOperand() 266 << " dst_sel:" << getDstSel() 267 << " preserve:" << *getPreservedOperand() << '\n'; 268 } 269 270 #endif 271 272 static void copyRegOperand(MachineOperand &To, const MachineOperand &From) { 273 assert(To.isReg() && From.isReg()); 274 To.setReg(From.getReg()); 275 To.setSubReg(From.getSubReg()); 276 To.setIsUndef(From.isUndef()); 277 if (To.isUse()) { 278 To.setIsKill(From.isKill()); 279 } else { 280 To.setIsDead(From.isDead()); 281 } 282 } 283 284 static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) { 285 return LHS.isReg() && 286 RHS.isReg() && 287 LHS.getReg() == RHS.getReg() && 288 LHS.getSubReg() == RHS.getSubReg(); 289 } 290 291 static MachineOperand *findSingleRegUse(const MachineOperand *Reg, 292 const MachineRegisterInfo *MRI) { 293 if (!Reg->isReg() || !Reg->isDef()) 294 return nullptr; 295 296 MachineOperand *ResMO = nullptr; 297 for (MachineOperand &UseMO : MRI->use_nodbg_operands(Reg->getReg())) { 298 // If there exist use of subreg of Reg then return nullptr 299 if (!isSameReg(UseMO, *Reg)) 300 return nullptr; 301 302 // Check that there is only one instruction that uses Reg 303 if (!ResMO) { 304 ResMO = &UseMO; 305 } else if (ResMO->getParent() != UseMO.getParent()) { 306 return nullptr; 307 } 308 } 309 310 return ResMO; 311 } 312 313 static MachineOperand *findSingleRegDef(const MachineOperand *Reg, 314 const MachineRegisterInfo *MRI) { 315 if (!Reg->isReg()) 316 return nullptr; 317 318 MachineInstr *DefInstr = MRI->getUniqueVRegDef(Reg->getReg()); 319 if (!DefInstr) 320 return nullptr; 321 322 for (auto &DefMO : DefInstr->defs()) { 323 if (DefMO.isReg() && DefMO.getReg() == Reg->getReg()) 324 return &DefMO; 325 } 326 327 llvm_unreachable("invalid reg"); 328 } 329 330 uint64_t SDWASrcOperand::getSrcMods(const SIInstrInfo *TII, 331 const MachineOperand *SrcOp) const { 332 uint64_t Mods = 0; 333 const auto *MI = SrcOp->getParent(); 334 if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) { 335 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) { 336 Mods = Mod->getImm(); 337 } 338 } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) { 339 if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) { 340 Mods = Mod->getImm(); 341 } 342 } 343 if (Abs || Neg) { 344 assert(!Sext && 345 "Float and integer src modifiers can't be set simulteniously"); 346 Mods |= Abs ? SISrcMods::ABS : 0; 347 Mods ^= Neg ? SISrcMods::NEG : 0; 348 } else if (Sext) { 349 Mods |= SISrcMods::SEXT; 350 } 351 352 return Mods; 353 } 354 355 MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII) { 356 // For SDWA src operand potential instruction is one that use register 357 // defined by parent instruction 358 MachineOperand *PotentialMO = findSingleRegUse(getReplacedOperand(), getMRI()); 359 if (!PotentialMO) 360 return nullptr; 361 362 return PotentialMO->getParent(); 363 } 364 365 bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) { 366 // Find operand in instruction that matches source operand and replace it with 367 // target operand. Set corresponding src_sel 368 369 MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 370 MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); 371 MachineOperand *SrcMods = 372 TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 373 assert(Src && (Src->isReg() || Src->isImm())); 374 if (!isSameReg(*Src, *getReplacedOperand())) { 375 // If this is not src0 then it should be src1 376 Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 377 SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); 378 SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 379 380 assert(Src && Src->isReg()); 381 382 if ((MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || 383 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && 384 !isSameReg(*Src, *getReplacedOperand())) { 385 // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to 386 // src2. This is not allowed. 387 return false; 388 } 389 390 assert(isSameReg(*Src, *getReplacedOperand()) && SrcSel && SrcMods); 391 } 392 copyRegOperand(*Src, *getTargetOperand()); 393 SrcSel->setImm(getSrcSel()); 394 SrcMods->setImm(getSrcMods(TII, Src)); 395 getTargetOperand()->setIsKill(false); 396 return true; 397 } 398 399 MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII) { 400 // For SDWA dst operand potential instruction is one that defines register 401 // that this operand uses 402 MachineRegisterInfo *MRI = getMRI(); 403 MachineInstr *ParentMI = getParentInst(); 404 405 MachineOperand *PotentialMO = findSingleRegDef(getReplacedOperand(), MRI); 406 if (!PotentialMO) 407 return nullptr; 408 409 // Check that ParentMI is the only instruction that uses replaced register 410 for (MachineInstr &UseInst : MRI->use_nodbg_instructions(PotentialMO->getReg())) { 411 if (&UseInst != ParentMI) 412 return nullptr; 413 } 414 415 return PotentialMO->getParent(); 416 } 417 418 bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) { 419 // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused 420 421 if ((MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa || 422 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) && 423 getDstSel() != AMDGPU::SDWA::DWORD) { 424 // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD 425 return false; 426 } 427 428 MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 429 assert(Operand && 430 Operand->isReg() && 431 isSameReg(*Operand, *getReplacedOperand())); 432 copyRegOperand(*Operand, *getTargetOperand()); 433 MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); 434 assert(DstSel); 435 DstSel->setImm(getDstSel()); 436 MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); 437 assert(DstUnused); 438 DstUnused->setImm(getDstUnused()); 439 440 // Remove original instruction because it would conflict with our new 441 // instruction by register definition 442 getParentInst()->eraseFromParent(); 443 return true; 444 } 445 446 bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI, 447 const SIInstrInfo *TII) { 448 // MI should be moved right before v_or_b32. 449 // For this we should clear all kill flags on uses of MI src-operands or else 450 // we can encounter problem with use of killed operand. 451 for (MachineOperand &MO : MI.uses()) { 452 if (!MO.isReg()) 453 continue; 454 getMRI()->clearKillFlags(MO.getReg()); 455 } 456 457 // Move MI before v_or_b32 458 auto MBB = MI.getParent(); 459 MBB->remove(&MI); 460 MBB->insert(getParentInst(), &MI); 461 462 // Add Implicit use of preserved register 463 MachineInstrBuilder MIB(*MBB->getParent(), MI); 464 MIB.addReg(getPreservedOperand()->getReg(), 465 RegState::ImplicitKill, 466 getPreservedOperand()->getSubReg()); 467 468 // Tie dst to implicit use 469 MI.tieOperands(AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst), 470 MI.getNumOperands() - 1); 471 472 // Convert MI as any other SDWADstOperand and remove v_or_b32 473 return SDWADstOperand::convertToSDWA(MI, TII); 474 } 475 476 Optional<int64_t> SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const { 477 if (Op.isImm()) { 478 return Op.getImm(); 479 } 480 481 // If this is not immediate then it can be copy of immediate value, e.g.: 482 // %1<def> = S_MOV_B32 255; 483 if (Op.isReg()) { 484 for (const MachineOperand &Def : MRI->def_operands(Op.getReg())) { 485 if (!isSameReg(Op, Def)) 486 continue; 487 488 const MachineInstr *DefInst = Def.getParent(); 489 if (!TII->isFoldableCopy(*DefInst)) 490 return None; 491 492 const MachineOperand &Copied = DefInst->getOperand(1); 493 if (!Copied.isImm()) 494 return None; 495 496 return Copied.getImm(); 497 } 498 } 499 500 return None; 501 } 502 503 std::unique_ptr<SDWAOperand> 504 SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) { 505 unsigned Opcode = MI.getOpcode(); 506 switch (Opcode) { 507 case AMDGPU::V_LSHRREV_B32_e32: 508 case AMDGPU::V_ASHRREV_I32_e32: 509 case AMDGPU::V_LSHLREV_B32_e32: 510 case AMDGPU::V_LSHRREV_B32_e64: 511 case AMDGPU::V_ASHRREV_I32_e64: 512 case AMDGPU::V_LSHLREV_B32_e64: { 513 // from: v_lshrrev_b32_e32 v1, 16/24, v0 514 // to SDWA src:v0 src_sel:WORD_1/BYTE_3 515 516 // from: v_ashrrev_i32_e32 v1, 16/24, v0 517 // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1 518 519 // from: v_lshlrev_b32_e32 v1, 16/24, v0 520 // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD 521 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 522 auto Imm = foldToImm(*Src0); 523 if (!Imm) 524 break; 525 526 if (*Imm != 16 && *Imm != 24) 527 break; 528 529 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 530 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 531 if (TRI->isPhysicalRegister(Src1->getReg()) || 532 TRI->isPhysicalRegister(Dst->getReg())) 533 break; 534 535 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 || 536 Opcode == AMDGPU::V_LSHLREV_B32_e64) { 537 return make_unique<SDWADstOperand>( 538 Dst, Src1, *Imm == 16 ? WORD_1 : BYTE_3, UNUSED_PAD); 539 } else { 540 return make_unique<SDWASrcOperand>( 541 Src1, Dst, *Imm == 16 ? WORD_1 : BYTE_3, false, false, 542 Opcode != AMDGPU::V_LSHRREV_B32_e32 && 543 Opcode != AMDGPU::V_LSHRREV_B32_e64); 544 } 545 break; 546 } 547 548 case AMDGPU::V_LSHRREV_B16_e32: 549 case AMDGPU::V_ASHRREV_I16_e32: 550 case AMDGPU::V_LSHLREV_B16_e32: 551 case AMDGPU::V_LSHRREV_B16_e64: 552 case AMDGPU::V_ASHRREV_I16_e64: 553 case AMDGPU::V_LSHLREV_B16_e64: { 554 // from: v_lshrrev_b16_e32 v1, 8, v0 555 // to SDWA src:v0 src_sel:BYTE_1 556 557 // from: v_ashrrev_i16_e32 v1, 8, v0 558 // to SDWA src:v0 src_sel:BYTE_1 sext:1 559 560 // from: v_lshlrev_b16_e32 v1, 8, v0 561 // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD 562 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 563 auto Imm = foldToImm(*Src0); 564 if (!Imm || *Imm != 8) 565 break; 566 567 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 568 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 569 570 if (TRI->isPhysicalRegister(Src1->getReg()) || 571 TRI->isPhysicalRegister(Dst->getReg())) 572 break; 573 574 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 || 575 Opcode == AMDGPU::V_LSHLREV_B16_e64) { 576 return make_unique<SDWADstOperand>(Dst, Src1, BYTE_1, UNUSED_PAD); 577 } else { 578 return make_unique<SDWASrcOperand>( 579 Src1, Dst, BYTE_1, false, false, 580 Opcode != AMDGPU::V_LSHRREV_B16_e32 && 581 Opcode != AMDGPU::V_LSHRREV_B16_e64); 582 } 583 break; 584 } 585 586 case AMDGPU::V_BFE_I32: 587 case AMDGPU::V_BFE_U32: { 588 // e.g.: 589 // from: v_bfe_u32 v1, v0, 8, 8 590 // to SDWA src:v0 src_sel:BYTE_1 591 592 // offset | width | src_sel 593 // ------------------------ 594 // 0 | 8 | BYTE_0 595 // 0 | 16 | WORD_0 596 // 0 | 32 | DWORD ? 597 // 8 | 8 | BYTE_1 598 // 16 | 8 | BYTE_2 599 // 16 | 16 | WORD_1 600 // 24 | 8 | BYTE_3 601 602 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 603 auto Offset = foldToImm(*Src1); 604 if (!Offset) 605 break; 606 607 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); 608 auto Width = foldToImm(*Src2); 609 if (!Width) 610 break; 611 612 SdwaSel SrcSel = DWORD; 613 614 if (*Offset == 0 && *Width == 8) 615 SrcSel = BYTE_0; 616 else if (*Offset == 0 && *Width == 16) 617 SrcSel = WORD_0; 618 else if (*Offset == 0 && *Width == 32) 619 SrcSel = DWORD; 620 else if (*Offset == 8 && *Width == 8) 621 SrcSel = BYTE_1; 622 else if (*Offset == 16 && *Width == 8) 623 SrcSel = BYTE_2; 624 else if (*Offset == 16 && *Width == 16) 625 SrcSel = WORD_1; 626 else if (*Offset == 24 && *Width == 8) 627 SrcSel = BYTE_3; 628 else 629 break; 630 631 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 632 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 633 634 if (TRI->isPhysicalRegister(Src0->getReg()) || 635 TRI->isPhysicalRegister(Dst->getReg())) 636 break; 637 638 return make_unique<SDWASrcOperand>( 639 Src0, Dst, SrcSel, false, false, Opcode != AMDGPU::V_BFE_U32); 640 } 641 642 case AMDGPU::V_AND_B32_e32: 643 case AMDGPU::V_AND_B32_e64: { 644 // e.g.: 645 // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0 646 // to SDWA src:v0 src_sel:WORD_0/BYTE_0 647 648 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 649 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 650 auto ValSrc = Src1; 651 auto Imm = foldToImm(*Src0); 652 653 if (!Imm) { 654 Imm = foldToImm(*Src1); 655 ValSrc = Src0; 656 } 657 658 if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff)) 659 break; 660 661 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 662 663 if (TRI->isPhysicalRegister(Src1->getReg()) || 664 TRI->isPhysicalRegister(Dst->getReg())) 665 break; 666 667 return make_unique<SDWASrcOperand>( 668 ValSrc, Dst, *Imm == 0x0000ffff ? WORD_0 : BYTE_0); 669 } 670 671 case AMDGPU::V_OR_B32_e32: 672 case AMDGPU::V_OR_B32_e64: { 673 // Patterns for dst_unused:UNUSED_PRESERVE. 674 // e.g., from: 675 // v_add_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD 676 // src1_sel:WORD_1 src2_sel:WORD1 677 // v_add_f16_e32 v3, v1, v2 678 // v_or_b32_e32 v4, v0, v3 679 // to SDWA preserve dst:v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE preserve:v3 680 681 // Check if one of operands of v_or_b32 is SDWA instruction 682 using CheckRetType = Optional<std::pair<MachineOperand *, MachineOperand *>>; 683 auto CheckOROperandsForSDWA = 684 [&](const MachineOperand *Op1, const MachineOperand *Op2) -> CheckRetType { 685 if (!Op1 || !Op1->isReg() || !Op2 || !Op2->isReg()) 686 return CheckRetType(None); 687 688 MachineOperand *Op1Def = findSingleRegDef(Op1, MRI); 689 if (!Op1Def) 690 return CheckRetType(None); 691 692 MachineInstr *Op1Inst = Op1Def->getParent(); 693 if (!TII->isSDWA(*Op1Inst)) 694 return CheckRetType(None); 695 696 MachineOperand *Op2Def = findSingleRegDef(Op2, MRI); 697 if (!Op2Def) 698 return CheckRetType(None); 699 700 return CheckRetType(std::make_pair(Op1Def, Op2Def)); 701 }; 702 703 MachineOperand *OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 704 MachineOperand *OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 705 assert(OrSDWA && OrOther); 706 auto Res = CheckOROperandsForSDWA(OrSDWA, OrOther); 707 if (!Res) { 708 OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 709 OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 710 assert(OrSDWA && OrOther); 711 Res = CheckOROperandsForSDWA(OrSDWA, OrOther); 712 if (!Res) 713 break; 714 } 715 716 MachineOperand *OrSDWADef = Res->first; 717 MachineOperand *OrOtherDef = Res->second; 718 assert(OrSDWADef && OrOtherDef); 719 720 MachineInstr *SDWAInst = OrSDWADef->getParent(); 721 MachineInstr *OtherInst = OrOtherDef->getParent(); 722 723 // Check that OtherInstr is actually bitwise compatible with SDWAInst = their 724 // destination patterns don't overlap. Compatible instruction can be either 725 // regular instruction with compatible bitness or SDWA instruction with 726 // correct dst_sel 727 // SDWAInst | OtherInst bitness / OtherInst dst_sel 728 // ----------------------------------------------------- 729 // DWORD | no / no 730 // WORD_0 | no / BYTE_2/3, WORD_1 731 // WORD_1 | 8/16-bit instructions / BYTE_0/1, WORD_0 732 // BYTE_0 | no / BYTE_1/2/3, WORD_1 733 // BYTE_1 | 8-bit / BYTE_0/2/3, WORD_1 734 // BYTE_2 | 8/16-bit / BYTE_0/1/3. WORD_0 735 // BYTE_3 | 8/16/24-bit / BYTE_0/1/2, WORD_0 736 // E.g. if SDWAInst is v_add_f16_sdwa dst_sel:WORD_1 then v_add_f16 is OK 737 // but v_add_f32 is not. 738 739 // TODO: add support for non-SDWA instructions as OtherInst. 740 // For now this only works with SDWA instructions. For regular instructions 741 // there is no way to determine if instruction write only 8/16/24-bit out of 742 // full register size and all registers are at min 32-bit wide. 743 if (!TII->isSDWA(*OtherInst)) 744 break; 745 746 SdwaSel DstSel = static_cast<SdwaSel>( 747 TII->getNamedImmOperand(*SDWAInst, AMDGPU::OpName::dst_sel));; 748 SdwaSel OtherDstSel = static_cast<SdwaSel>( 749 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_sel)); 750 751 bool DstSelAgree = false; 752 switch (DstSel) { 753 case WORD_0: DstSelAgree = ((OtherDstSel == BYTE_2) || 754 (OtherDstSel == BYTE_3) || 755 (OtherDstSel == WORD_1)); 756 break; 757 case WORD_1: DstSelAgree = ((OtherDstSel == BYTE_0) || 758 (OtherDstSel == BYTE_1) || 759 (OtherDstSel == WORD_0)); 760 break; 761 case BYTE_0: DstSelAgree = ((OtherDstSel == BYTE_1) || 762 (OtherDstSel == BYTE_2) || 763 (OtherDstSel == BYTE_3) || 764 (OtherDstSel == WORD_1)); 765 break; 766 case BYTE_1: DstSelAgree = ((OtherDstSel == BYTE_0) || 767 (OtherDstSel == BYTE_2) || 768 (OtherDstSel == BYTE_3) || 769 (OtherDstSel == WORD_1)); 770 break; 771 case BYTE_2: DstSelAgree = ((OtherDstSel == BYTE_0) || 772 (OtherDstSel == BYTE_1) || 773 (OtherDstSel == BYTE_3) || 774 (OtherDstSel == WORD_0)); 775 break; 776 case BYTE_3: DstSelAgree = ((OtherDstSel == BYTE_0) || 777 (OtherDstSel == BYTE_1) || 778 (OtherDstSel == BYTE_2) || 779 (OtherDstSel == WORD_0)); 780 break; 781 default: DstSelAgree = false; 782 } 783 784 if (!DstSelAgree) 785 break; 786 787 // Also OtherInst dst_unused should be UNUSED_PAD 788 DstUnused OtherDstUnused = static_cast<DstUnused>( 789 TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_unused)); 790 if (OtherDstUnused != DstUnused::UNUSED_PAD) 791 break; 792 793 // Create DstPreserveOperand 794 MachineOperand *OrDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 795 assert(OrDst && OrDst->isReg()); 796 797 return make_unique<SDWADstPreserveOperand>( 798 OrDst, OrSDWADef, OrOtherDef, DstSel); 799 800 } 801 } 802 803 return std::unique_ptr<SDWAOperand>(nullptr); 804 } 805 806 void SIPeepholeSDWA::matchSDWAOperands(MachineFunction &MF) { 807 for (MachineBasicBlock &MBB : MF) { 808 for (MachineInstr &MI : MBB) { 809 if (auto Operand = matchSDWAOperand(MI)) { 810 DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n'); 811 SDWAOperands[&MI] = std::move(Operand); 812 ++NumSDWAPatternsFound; 813 } 814 } 815 } 816 } 817 818 bool SIPeepholeSDWA::isConvertibleToSDWA(const MachineInstr &MI, 819 const SISubtarget &ST) const { 820 // Check if this is already an SDWA instruction 821 unsigned Opc = MI.getOpcode(); 822 if (TII->isSDWA(Opc)) 823 return true; 824 825 // Check if this instruction has opcode that supports SDWA 826 if (AMDGPU::getSDWAOp(Opc) == -1) 827 Opc = AMDGPU::getVOPe32(Opc); 828 829 if (AMDGPU::getSDWAOp(Opc) == -1) 830 return false; 831 832 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) 833 return false; 834 835 if (TII->isVOPC(Opc)) { 836 if (!ST.hasSDWASdst()) { 837 const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); 838 if (SDst && SDst->getReg() != AMDGPU::VCC) 839 return false; 840 } 841 842 if (!ST.hasSDWAOutModsVOPC() && 843 (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) || 844 TII->hasModifiersSet(MI, AMDGPU::OpName::omod))) 845 return false; 846 847 } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) || 848 !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) { 849 return false; 850 } 851 852 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_MAC_F16_e32 || 853 Opc == AMDGPU::V_MAC_F32_e32)) 854 return false; 855 856 return true; 857 } 858 859 bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI, 860 const SDWAOperandsVector &SDWAOperands) { 861 // Convert to sdwa 862 int SDWAOpcode; 863 unsigned Opcode = MI.getOpcode(); 864 if (TII->isSDWA(Opcode)) { 865 SDWAOpcode = Opcode; 866 } else { 867 SDWAOpcode = AMDGPU::getSDWAOp(Opcode); 868 if (SDWAOpcode == -1) 869 SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(Opcode)); 870 } 871 assert(SDWAOpcode != -1); 872 873 const MCInstrDesc &SDWADesc = TII->get(SDWAOpcode); 874 875 // Create SDWA version of instruction MI and initialize its operands 876 MachineInstrBuilder SDWAInst = 877 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), SDWADesc); 878 879 // Copy dst, if it is present in original then should also be present in SDWA 880 MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst); 881 if (Dst) { 882 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst) != -1); 883 SDWAInst.add(*Dst); 884 } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) { 885 assert(Dst && 886 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1); 887 SDWAInst.add(*Dst); 888 } else { 889 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1); 890 SDWAInst.addReg(AMDGPU::VCC, RegState::Define); 891 } 892 893 // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and 894 // src0_modifiers (except for v_nop_sdwa, but it can't get here) 895 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 896 assert( 897 Src0 && 898 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0) != -1 && 899 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_modifiers) != -1); 900 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)) 901 SDWAInst.addImm(Mod->getImm()); 902 else 903 SDWAInst.addImm(0); 904 SDWAInst.add(*Src0); 905 906 // Copy src1 if present, initialize src1_modifiers. 907 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 908 if (Src1) { 909 assert( 910 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1) != -1 && 911 AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_modifiers) != -1); 912 if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)) 913 SDWAInst.addImm(Mod->getImm()); 914 else 915 SDWAInst.addImm(0); 916 SDWAInst.add(*Src1); 917 } 918 919 if (SDWAOpcode == AMDGPU::V_MAC_F16_sdwa || 920 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) { 921 // v_mac_f16/32 has additional src2 operand tied to vdst 922 MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2); 923 assert(Src2); 924 SDWAInst.add(*Src2); 925 } 926 927 // Copy clamp if present, initialize otherwise 928 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::clamp) != -1); 929 MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp); 930 if (Clamp) { 931 SDWAInst.add(*Clamp); 932 } else { 933 SDWAInst.addImm(0); 934 } 935 936 // Copy omod if present, initialize otherwise if needed 937 if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::omod) != -1) { 938 MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod); 939 if (OMod) { 940 SDWAInst.add(*OMod); 941 } else { 942 SDWAInst.addImm(0); 943 } 944 } 945 946 // Copy dst_sel if present, initialize otherwise if needed 947 if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_sel) != -1) { 948 MachineOperand *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel); 949 if (DstSel) { 950 SDWAInst.add(*DstSel); 951 } else { 952 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); 953 } 954 } 955 956 // Copy dst_unused if present, initialize otherwise if needed 957 if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_unused) != -1) { 958 MachineOperand *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused); 959 if (DstUnused) { 960 SDWAInst.add(*DstUnused); 961 } else { 962 SDWAInst.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PAD); 963 } 964 } 965 966 // Copy src0_sel if present, initialize otherwise 967 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_sel) != -1); 968 MachineOperand *Src0Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel); 969 if (Src0Sel) { 970 SDWAInst.add(*Src0Sel); 971 } else { 972 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); 973 } 974 975 // Copy src1_sel if present, initialize otherwise if needed 976 if (Src1) { 977 assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_sel) != -1); 978 MachineOperand *Src1Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel); 979 if (Src1Sel) { 980 SDWAInst.add(*Src1Sel); 981 } else { 982 SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD); 983 } 984 } 985 986 // Apply all sdwa operand pattenrs 987 bool Converted = false; 988 for (auto &Operand : SDWAOperands) { 989 // There should be no intesection between SDWA operands and potential MIs 990 // e.g.: 991 // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0 992 // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0 993 // v_add_u32 v3, v4, v2 994 // 995 // In that example it is possible that we would fold 2nd instruction into 3rd 996 // (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that was 997 // already destroyed). So if SDWAOperand is also a potential MI then do not 998 // apply it. 999 if (PotentialMatches.count(Operand->getParentInst()) == 0) 1000 Converted |= Operand->convertToSDWA(*SDWAInst, TII); 1001 } 1002 if (Converted) { 1003 ConvertedInstructions.push_back(SDWAInst); 1004 } else { 1005 SDWAInst->eraseFromParent(); 1006 return false; 1007 } 1008 1009 DEBUG(dbgs() << "Convert instruction:" << MI 1010 << "Into:" << *SDWAInst << '\n'); 1011 ++NumSDWAInstructionsPeepholed; 1012 1013 MI.eraseFromParent(); 1014 return true; 1015 } 1016 1017 // If an instruction was converted to SDWA it should not have immediates or SGPR 1018 // operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs. 1019 void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI, const SISubtarget &ST) const { 1020 const MCInstrDesc &Desc = TII->get(MI.getOpcode()); 1021 unsigned ConstantBusCount = 0; 1022 for (MachineOperand &Op : MI.explicit_uses()) { 1023 if (!Op.isImm() && !(Op.isReg() && !TRI->isVGPR(*MRI, Op.getReg()))) 1024 continue; 1025 1026 unsigned I = MI.getOperandNo(&Op); 1027 if (Desc.OpInfo[I].RegClass == -1 || 1028 !TRI->hasVGPRs(TRI->getRegClass(Desc.OpInfo[I].RegClass))) 1029 continue; 1030 1031 if (ST.hasSDWAScalar() && ConstantBusCount == 0 && Op.isReg() && 1032 TRI->isSGPRReg(*MRI, Op.getReg())) { 1033 ++ConstantBusCount; 1034 continue; 1035 } 1036 1037 unsigned VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1038 auto Copy = BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(), 1039 TII->get(AMDGPU::V_MOV_B32_e32), VGPR); 1040 if (Op.isImm()) 1041 Copy.addImm(Op.getImm()); 1042 else if (Op.isReg()) 1043 Copy.addReg(Op.getReg(), Op.isKill() ? RegState::Kill : 0, 1044 Op.getSubReg()); 1045 Op.ChangeToRegister(VGPR, false); 1046 } 1047 } 1048 1049 bool SIPeepholeSDWA::runOnMachineFunction(MachineFunction &MF) { 1050 const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); 1051 1052 if (!ST.hasSDWA() || skipFunction(*MF.getFunction())) 1053 return false; 1054 1055 MRI = &MF.getRegInfo(); 1056 TRI = ST.getRegisterInfo(); 1057 TII = ST.getInstrInfo(); 1058 1059 // Find all SDWA operands in MF. 1060 bool Changed = false; 1061 bool Ret = false; 1062 do { 1063 matchSDWAOperands(MF); 1064 1065 for (const auto &OperandPair : SDWAOperands) { 1066 const auto &Operand = OperandPair.second; 1067 MachineInstr *PotentialMI = Operand->potentialToConvert(TII); 1068 if (PotentialMI && isConvertibleToSDWA(*PotentialMI, ST)) { 1069 PotentialMatches[PotentialMI].push_back(Operand.get()); 1070 } 1071 } 1072 1073 for (auto &PotentialPair : PotentialMatches) { 1074 MachineInstr &PotentialMI = *PotentialPair.first; 1075 convertToSDWA(PotentialMI, PotentialPair.second); 1076 } 1077 1078 PotentialMatches.clear(); 1079 SDWAOperands.clear(); 1080 1081 Changed = !ConvertedInstructions.empty(); 1082 1083 if (Changed) 1084 Ret = true; 1085 1086 while (!ConvertedInstructions.empty()) 1087 legalizeScalarOperands(*ConvertedInstructions.pop_back_val(), ST); 1088 } while (Changed); 1089 1090 return Ret; 1091 } 1092