Lines Matching defs:AMDGPU
15 #include "AMDGPU.h"
43 namespace llvm::AMDGPU {
48 } // namespace llvm::AMDGPU
64 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
86 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
87 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
159 case AMDGPU::S_AND_SAVEEXEC_B32:
160 case AMDGPU::S_AND_SAVEEXEC_B64:
162 case AMDGPU::S_AND_B32:
163 case AMDGPU::S_AND_B64:
164 if (!Use.readsRegister(AMDGPU::EXEC, /*TRI=*/nullptr))
177 case AMDGPU::V_READFIRSTLANE_B32:
186 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
194 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
260 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
261 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
278 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) ||
279 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase))
312 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
313 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
314 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
317 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
318 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
346 case AMDGPU::DS_READ2ST64_B32:
347 case AMDGPU::DS_READ2ST64_B64:
348 case AMDGPU::DS_WRITE2ST64_B32:
349 case AMDGPU::DS_WRITE2ST64_B64:
369 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
370 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
381 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
383 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
390 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
392 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
407 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
417 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
419 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
421 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
431 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
435 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
439 getNamedOperand(LdSt, AMDGPU::OpName::offset);
442 getNamedOperand(LdSt, AMDGPU::OpName::soffset);
450 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
452 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
461 isMIMG(LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
462 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, RsrcOpName);
464 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
470 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
474 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
482 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
486 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
489 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
498 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
501 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
504 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
506 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
508 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
623 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
642 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
643 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
646 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
662 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
684 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
703 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
708 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
717 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI,
727 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
728 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
729 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
731 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
742 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
763 unsigned Opcode = AMDGPU::S_MOV_B32;
766 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
767 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
775 Opcode = AMDGPU::S_MOV_B64;
819 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16);
825 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
835 if (RC == &AMDGPU::VGPR_32RegClass) {
836 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
837 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
838 AMDGPU::AGPR_32RegClass.contains(SrcReg));
839 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
840 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
846 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
847 RC == &AMDGPU::SReg_32RegClass) {
848 if (SrcReg == AMDGPU::SCC) {
849 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
855 if (DestReg == AMDGPU::VCC_LO) {
856 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
857 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
861 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
862 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
870 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
875 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
880 if (RC == &AMDGPU::SReg_64RegClass) {
881 if (SrcReg == AMDGPU::SCC) {
882 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
888 if (DestReg == AMDGPU::VCC) {
889 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
890 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
894 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
895 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
903 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
908 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
913 if (DestReg == AMDGPU::SCC) {
916 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
921 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))
925 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
926 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
934 if (RC == &AMDGPU::AGPR_32RegClass) {
935 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
936 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {
937 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
942 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {
943 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
957 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
958 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
959 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
961 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
962 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
963 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
964 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
965 bool DstLow = !AMDGPU::isHi16Reg(DestReg, RI);
966 bool SrcLow = !AMDGPU::isHi16Reg(SrcReg, RI);
976 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)
997 if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) &&
998 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) {
999 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg)
1002 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg)
1016 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
1021 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
1025 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1026 : AMDGPU::SDWA::SdwaSel::WORD_1)
1027 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1028 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1029 : AMDGPU::SDWA::SdwaSel::WORD_1)
1038 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)
1043 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
1071 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1074 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1077 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1079 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1081 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1087 Opcode = AMDGPU::V_MOV_B64_e32;
1090 Opcode = AMDGPU::V_PK_MOV_B32;
1101 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1124 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1129 } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1131 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg)
1159 NewOpc = AMDGPU::getCommuteRev(Opcode);
1165 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1179 if (RegClass == &AMDGPU::SReg_32RegClass ||
1180 RegClass == &AMDGPU::SGPR_32RegClass ||
1181 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
1182 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
1183 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
1188 if (RegClass == &AMDGPU::SReg_64RegClass ||
1189 RegClass == &AMDGPU::SGPR_64RegClass ||
1190 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
1191 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
1196 if (RegClass == &AMDGPU::VGPR_32RegClass) {
1197 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
1201 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) {
1202 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
1208 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1211 Opcode = AMDGPU::S_MOV_B64;
1214 Opcode = AMDGPU::S_MOV_B32;
1231 return &AMDGPU::VGPR_32RegClass;
1242 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
1247 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1249 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1260 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1261 : AMDGPU::S_CSELECT_B64), SReg)
1264 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1274 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1275 : AMDGPU::S_CSELECT_B64), SReg)
1278 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1290 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1292 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1304 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1306 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1317 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1318 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1320 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1321 : AMDGPU::S_CSELECT_B64), SReg)
1324 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1335 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1336 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1338 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1339 : AMDGPU::S_CSELECT_B64), SReg)
1342 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1365 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
1378 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
1388 return AMDGPU::COPY;
1392 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1395 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1397 return AMDGPU::S_MOV_B64;
1399 return AMDGPU::V_MOV_B64_PSEUDO;
1400 return AMDGPU::COPY;
1408 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1410 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1412 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1414 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1416 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1418 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1420 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1422 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1424 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1426 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1428 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1430 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1436 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1438 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1440 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1442 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1444 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1446 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1448 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1450 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1452 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1454 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1456 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1458 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1465 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1467 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1469 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1471 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1473 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1475 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1477 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1479 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1481 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1483 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1485 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1487 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1494 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1496 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1498 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1500 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1502 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1504 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1506 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1508 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1510 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1512 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1514 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1516 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1523 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1525 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1527 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1529 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1531 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1557 return AMDGPU::SI_SPILL_S32_SAVE;
1559 return AMDGPU::SI_SPILL_S64_SAVE;
1561 return AMDGPU::SI_SPILL_S96_SAVE;
1563 return AMDGPU::SI_SPILL_S128_SAVE;
1565 return AMDGPU::SI_SPILL_S160_SAVE;
1567 return AMDGPU::SI_SPILL_S192_SAVE;
1569 return AMDGPU::SI_SPILL_S224_SAVE;
1571 return AMDGPU::SI_SPILL_S256_SAVE;
1573 return AMDGPU::SI_SPILL_S288_SAVE;
1575 return AMDGPU::SI_SPILL_S320_SAVE;
1577 return AMDGPU::SI_SPILL_S352_SAVE;
1579 return AMDGPU::SI_SPILL_S384_SAVE;
1581 return AMDGPU::SI_SPILL_S512_SAVE;
1583 return AMDGPU::SI_SPILL_S1024_SAVE;
1592 return AMDGPU::SI_SPILL_V32_SAVE;
1594 return AMDGPU::SI_SPILL_V64_SAVE;
1596 return AMDGPU::SI_SPILL_V96_SAVE;
1598 return AMDGPU::SI_SPILL_V128_SAVE;
1600 return AMDGPU::SI_SPILL_V160_SAVE;
1602 return AMDGPU::SI_SPILL_V192_SAVE;
1604 return AMDGPU::SI_SPILL_V224_SAVE;
1606 return AMDGPU::SI_SPILL_V256_SAVE;
1608 return AMDGPU::SI_SPILL_V288_SAVE;
1610 return AMDGPU::SI_SPILL_V320_SAVE;
1612 return AMDGPU::SI_SPILL_V352_SAVE;
1614 return AMDGPU::SI_SPILL_V384_SAVE;
1616 return AMDGPU::SI_SPILL_V512_SAVE;
1618 return AMDGPU::SI_SPILL_V1024_SAVE;
1627 return AMDGPU::SI_SPILL_A32_SAVE;
1629 return AMDGPU::SI_SPILL_A64_SAVE;
1631 return AMDGPU::SI_SPILL_A96_SAVE;
1633 return AMDGPU::SI_SPILL_A128_SAVE;
1635 return AMDGPU::SI_SPILL_A160_SAVE;
1637 return AMDGPU::SI_SPILL_A192_SAVE;
1639 return AMDGPU::SI_SPILL_A224_SAVE;
1641 return AMDGPU::SI_SPILL_A256_SAVE;
1643 return AMDGPU::SI_SPILL_A288_SAVE;
1645 return AMDGPU::SI_SPILL_A320_SAVE;
1647 return AMDGPU::SI_SPILL_A352_SAVE;
1649 return AMDGPU::SI_SPILL_A384_SAVE;
1651 return AMDGPU::SI_SPILL_A512_SAVE;
1653 return AMDGPU::SI_SPILL_A1024_SAVE;
1662 return AMDGPU::SI_SPILL_AV32_SAVE;
1664 return AMDGPU::SI_SPILL_AV64_SAVE;
1666 return AMDGPU::SI_SPILL_AV96_SAVE;
1668 return AMDGPU::SI_SPILL_AV128_SAVE;
1670 return AMDGPU::SI_SPILL_AV160_SAVE;
1672 return AMDGPU::SI_SPILL_AV192_SAVE;
1674 return AMDGPU::SI_SPILL_AV224_SAVE;
1676 return AMDGPU::SI_SPILL_AV256_SAVE;
1678 return AMDGPU::SI_SPILL_AV288_SAVE;
1680 return AMDGPU::SI_SPILL_AV320_SAVE;
1682 return AMDGPU::SI_SPILL_AV352_SAVE;
1684 return AMDGPU::SI_SPILL_AV384_SAVE;
1686 return AMDGPU::SI_SPILL_AV512_SAVE;
1688 return AMDGPU::SI_SPILL_AV1024_SAVE;
1701 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1703 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1714 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))
1744 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1745 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1746 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1755 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1784 return AMDGPU::SI_SPILL_S32_RESTORE;
1786 return AMDGPU::SI_SPILL_S64_RESTORE;
1788 return AMDGPU::SI_SPILL_S96_RESTORE;
1790 return AMDGPU::SI_SPILL_S128_RESTORE;
1792 return AMDGPU::SI_SPILL_S160_RESTORE;
1794 return AMDGPU::SI_SPILL_S192_RESTORE;
1796 return AMDGPU::SI_SPILL_S224_RESTORE;
1798 return AMDGPU::SI_SPILL_S256_RESTORE;
1800 return AMDGPU::SI_SPILL_S288_RESTORE;
1802 return AMDGPU::SI_SPILL_S320_RESTORE;
1804 return AMDGPU::SI_SPILL_S352_RESTORE;
1806 return AMDGPU::SI_SPILL_S384_RESTORE;
1808 return AMDGPU::SI_SPILL_S512_RESTORE;
1810 return AMDGPU::SI_SPILL_S1024_RESTORE;
1819 return AMDGPU::SI_SPILL_V32_RESTORE;
1821 return AMDGPU::SI_SPILL_V64_RESTORE;
1823 return AMDGPU::SI_SPILL_V96_RESTORE;
1825 return AMDGPU::SI_SPILL_V128_RESTORE;
1827 return AMDGPU::SI_SPILL_V160_RESTORE;
1829 return AMDGPU::SI_SPILL_V192_RESTORE;
1831 return AMDGPU::SI_SPILL_V224_RESTORE;
1833 return AMDGPU::SI_SPILL_V256_RESTORE;
1835 return AMDGPU::SI_SPILL_V288_RESTORE;
1837 return AMDGPU::SI_SPILL_V320_RESTORE;
1839 return AMDGPU::SI_SPILL_V352_RESTORE;
1841 return AMDGPU::SI_SPILL_V384_RESTORE;
1843 return AMDGPU::SI_SPILL_V512_RESTORE;
1845 return AMDGPU::SI_SPILL_V1024_RESTORE;
1854 return AMDGPU::SI_SPILL_A32_RESTORE;
1856 return AMDGPU::SI_SPILL_A64_RESTORE;
1858 return AMDGPU::SI_SPILL_A96_RESTORE;
1860 return AMDGPU::SI_SPILL_A128_RESTORE;
1862 return AMDGPU::SI_SPILL_A160_RESTORE;
1864 return AMDGPU::SI_SPILL_A192_RESTORE;
1866 return AMDGPU::SI_SPILL_A224_RESTORE;
1868 return AMDGPU::SI_SPILL_A256_RESTORE;
1870 return AMDGPU::SI_SPILL_A288_RESTORE;
1872 return AMDGPU::SI_SPILL_A320_RESTORE;
1874 return AMDGPU::SI_SPILL_A352_RESTORE;
1876 return AMDGPU::SI_SPILL_A384_RESTORE;
1878 return AMDGPU::SI_SPILL_A512_RESTORE;
1880 return AMDGPU::SI_SPILL_A1024_RESTORE;
1889 return AMDGPU::SI_SPILL_AV32_RESTORE;
1891 return AMDGPU::SI_SPILL_AV64_RESTORE;
1893 return AMDGPU::SI_SPILL_AV96_RESTORE;
1895 return AMDGPU::SI_SPILL_AV128_RESTORE;
1897 return AMDGPU::SI_SPILL_AV160_RESTORE;
1899 return AMDGPU::SI_SPILL_AV192_RESTORE;
1901 return AMDGPU::SI_SPILL_AV224_RESTORE;
1903 return AMDGPU::SI_SPILL_AV256_RESTORE;
1905 return AMDGPU::SI_SPILL_AV288_RESTORE;
1907 return AMDGPU::SI_SPILL_AV320_RESTORE;
1909 return AMDGPU::SI_SPILL_AV352_RESTORE;
1911 return AMDGPU::SI_SPILL_AV384_RESTORE;
1913 return AMDGPU::SI_SPILL_AV512_RESTORE;
1915 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1928 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1930 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1940 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG))
1972 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1973 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1974 DestReg != AMDGPU::EXEC && "exec should not be spilled");
1981 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
2015 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);
2029 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
2031 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
2052 BuildMI(MBB, MI, DL, get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(TrapBB);
2059 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_TRAP))
2061 Register DoorbellReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2062 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_SENDMSG_RTN_B32),
2064 .addImm(AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
2065 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::TTMP2)
2066 .addUse(AMDGPU::M0);
2068 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2069 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_AND_B32), DoorbellRegMasked)
2073 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
2074 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_OR_B32), SetWaveAbortBit)
2077 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2079 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_SENDMSG))
2080 .addImm(AMDGPU::SendMsg::ID_INTERRUPT);
2081 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_MOV_B32), AMDGPU::M0)
2082 .addUse(AMDGPU::TTMP2);
2083 BuildMI(*TrapBB, TrapBB->end(), DL, get(AMDGPU::S_BRANCH)).addMBB(HaltLoopBB);
2086 BuildMI(*HaltLoopBB, HaltLoopBB->end(), DL, get(AMDGPU::S_SETHALT)).addImm(5);
2087 BuildMI(*HaltLoopBB, HaltLoopBB->end(), DL, get(AMDGPU::S_BRANCH))
2102 case AMDGPU::S_NOP:
2114 case AMDGPU::S_MOV_B64_term:
2117 MI.setDesc(get(AMDGPU::S_MOV_B64));
2120 case AMDGPU::S_MOV_B32_term:
2123 MI.setDesc(get(AMDGPU::S_MOV_B32));
2126 case AMDGPU::S_XOR_B64_term:
2129 MI.setDesc(get(AMDGPU::S_XOR_B64));
2132 case AMDGPU::S_XOR_B32_term:
2135 MI.setDesc(get(AMDGPU::S_XOR_B32));
2137 case AMDGPU::S_OR_B64_term:
2140 MI.setDesc(get(AMDGPU::S_OR_B64));
2142 case AMDGPU::S_OR_B32_term:
2145 MI.setDesc(get(AMDGPU::S_OR_B32));
2148 case AMDGPU::S_ANDN2_B64_term:
2151 MI.setDesc(get(AMDGPU::S_ANDN2_B64));
2154 case AMDGPU::S_ANDN2_B32_term:
2157 MI.setDesc(get(AMDGPU::S_ANDN2_B32));
2160 case AMDGPU::S_AND_B64_term:
2163 MI.setDesc(get(AMDGPU::S_AND_B64));
2166 case AMDGPU::S_AND_B32_term:
2169 MI.setDesc(get(AMDGPU::S_AND_B32));
2172 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2175 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64));
2178 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2181 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32));
2184 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2185 MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
2188 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2189 MI.setDesc(get(AMDGPU::V_READLANE_B32));
2192 case AMDGPU::V_MOV_B64_PSEUDO: {
2194 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2195 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2201 MI.setDesc(get(AMDGPU::V_MOV_B64_e32));
2211 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
2222 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2225 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2233 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
2244 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2245 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
2247 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2248 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
2255 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2259 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2264 MI.setDesc(get(AMDGPU::S_MOV_B64));
2269 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2270 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2274 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)
2277 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)
2283 case AMDGPU::V_SET_INACTIVE_B32: {
2286 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
2295 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2296 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2297 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2298 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2299 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2300 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2301 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2302 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2303 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2304 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2305 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2306 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2307 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2308 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2309 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2310 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2311 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2312 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2313 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2314 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2315 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2316 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2317 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2318 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2319 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2320 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2321 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2322 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2323 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2328 Opc = AMDGPU::V_MOVRELD_B32_e32;
2330 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2331 : AMDGPU::S_MOVRELD_B32;
2354 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2355 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2356 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2357 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2358 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2359 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2360 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2361 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2362 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2363 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2364 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2365 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2372 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2374 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);
2377 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write);
2391 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2398 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2399 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2400 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2401 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2402 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2403 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2404 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2405 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2406 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2407 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2408 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2409 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2417 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2419 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2422 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read))
2427 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2434 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2437 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
2438 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
2445 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
2461 BuildMI(MF, DL, get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi));
2468 BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo));
2472 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
2481 case AMDGPU::ENTER_STRICT_WWM: {
2484 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
2485 : AMDGPU::S_OR_SAVEEXEC_B64));
2488 case AMDGPU::ENTER_STRICT_WQM: {
2491 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2492 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
2493 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
2500 case AMDGPU::EXIT_STRICT_WWM:
2501 case AMDGPU::EXIT_STRICT_WQM: {
2504 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
2507 case AMDGPU::SI_RETURN: {
2517 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return))
2525 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2526 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2527 MI.setDesc(get(AMDGPU::S_MUL_U64));
2530 case AMDGPU::S_GETPC_B64_pseudo:
2531 MI.setDesc(get(AMDGPU::S_GETPC_B64));
2534 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2537 BuildMI(MBB, std::next(MI.getIterator()), DL, get(AMDGPU::S_SEXT_I32_I16),
2556 case AMDGPU::S_LOAD_DWORDX16_IMM:
2557 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2579 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2591 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2593 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2603 UseMO->setSubReg(AMDGPU::NoSubRegister);
2609 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister);
2611 MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset);
2634 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2637 AMDGPU::isLegalDPALU_DPPControl(
2638 getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
2639 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
2651 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2652 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
2657 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2686 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
2688 .addImm(AMDGPU::sub0)
2690 .addImm(AMDGPU::sub1);
2698 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2779 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2824 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2826 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2852 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
2853 Src1, AMDGPU::OpName::src1_modifiers);
2855 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_sel, Src1,
2856 AMDGPU::OpName::src1_sel);
2880 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2884 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
2895 assert(BranchOp != AMDGPU::S_SETPC_B64);
2914 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
2915 MI.getOpcode() == AMDGPU::SI_LOOP)
2939 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
2949 BuildMI(MBB, I, DL, get(AMDGPU::S_WAITCNT_DEPCTR))
2950 .addImm(AMDGPU::DepCtr::encodeFieldSaSdst(0));
2955 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
2967 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
2968 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
2969 .addReg(PCReg, 0, AMDGPU::sub0)
2971 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
2972 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
2973 .addReg(PCReg, 0, AMDGPU::sub1)
2978 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
3022 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC),
3034 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);
3035 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);
3054 return AMDGPU::S_CBRANCH_SCC1;
3056 return AMDGPU::S_CBRANCH_SCC0;
3058 return AMDGPU::S_CBRANCH_VCCNZ;
3060 return AMDGPU::S_CBRANCH_VCCZ;
3062 return AMDGPU::S_CBRANCH_EXECNZ;
3064 return AMDGPU::S_CBRANCH_EXECZ;
3072 case AMDGPU::S_CBRANCH_SCC0:
3074 case AMDGPU::S_CBRANCH_SCC1:
3076 case AMDGPU::S_CBRANCH_VCCNZ:
3078 case AMDGPU::S_CBRANCH_VCCZ:
3080 case AMDGPU::S_CBRANCH_EXECNZ:
3082 case AMDGPU::S_CBRANCH_EXECZ:
3095 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3117 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3139 case AMDGPU::S_MOV_B64_term:
3140 case AMDGPU::S_XOR_B64_term:
3141 case AMDGPU::S_OR_B64_term:
3142 case AMDGPU::S_ANDN2_B64_term:
3143 case AMDGPU::S_AND_B64_term:
3144 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3145 case AMDGPU::S_MOV_B32_term:
3146 case AMDGPU::S_XOR_B32_term:
3147 case AMDGPU::S_OR_B32_term:
3148 case AMDGPU::S_ANDN2_B32_term:
3149 case AMDGPU::S_AND_B32_term:
3150 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3152 case AMDGPU::SI_IF:
3153 case AMDGPU::SI_ELSE:
3154 case AMDGPU::SI_KILL_I1_TERMINATOR:
3155 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3204 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
3236 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
3276 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
3291 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
3322 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)
3327 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)
3338 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
3347 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3348 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3349 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3350 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3354 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3355 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3356 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3357 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3360 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3361 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3369 SelOp = AMDGPU::S_CSELECT_B32;
3370 EltRC = &AMDGPU::SGPR_32RegClass;
3372 SelOp = AMDGPU::S_CSELECT_B64;
3373 EltRC = &AMDGPU::SGPR_64RegClass;
3380 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
3392 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3414 case AMDGPU::V_MOV_B16_t16_e32:
3415 case AMDGPU::V_MOV_B16_t16_e64:
3416 case AMDGPU::V_MOV_B32_e32:
3417 case AMDGPU::V_MOV_B32_e64:
3418 case AMDGPU::V_MOV_B64_PSEUDO:
3419 case AMDGPU::V_MOV_B64_e32:
3420 case AMDGPU::V_MOV_B64_e64:
3421 case AMDGPU::S_MOV_B32:
3422 case AMDGPU::S_MOV_B64:
3423 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3424 case AMDGPU::COPY:
3425 case AMDGPU::WWM_COPY:
3426 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3427 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3428 case AMDGPU::V_ACCVGPR_MOV_B32:
3436 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3437 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3438 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3443 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name);
3457 case AMDGPU::V_MOV_B64_e32:
3458 case AMDGPU::S_MOV_B64:
3459 case AMDGPU::V_MOV_B64_PSEUDO:
3460 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3461 case AMDGPU::V_MOV_B32_e32:
3462 case AMDGPU::S_MOV_B32:
3463 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3467 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
3478 case AMDGPU::sub0:
3480 case AMDGPU::sub1:
3482 case AMDGPU::lo16:
3484 case AMDGPU::hi16:
3486 case AMDGPU::sub1_lo16:
3488 case AMDGPU::sub1_hi16:
3496 if (Opc == AMDGPU::COPY) {
3504 unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO
3505 : AMDGPU::V_MOV_B32_e32
3506 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO
3507 : AMDGPU::S_MOV_B32;
3514 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
3521 if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16)
3543 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3544 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3545 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3546 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3547 Opc == AMDGPU::V_FMAC_F16_fake16_e64) {
3556 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
3562 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3563 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64;
3565 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3566 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3567 Opc == AMDGPU::V_FMAC_F16_fake16_e64;
3568 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
3569 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
3599 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32
3600 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_fake16
3601 : AMDGPU::V_FMAMK_F16)
3602 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
3609 if (NewOpc == AMDGPU::V_FMAMK_F16_fake16)
3623 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3624 Opc == AMDGPU::V_FMAC_F32_e64 ||
3625 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMAC_F16_e64)
3627 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3678 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32
3679 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_fake16
3680 : AMDGPU::V_FMAAK_F16)
3681 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
3688 if (NewOpc == AMDGPU::V_FMAAK_F16_fake16)
3694 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3695 Opc == AMDGPU::V_FMAC_F32_e64 ||
3696 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMAC_F16_e64)
3698 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3868 case AMDGPU::V_MAC_F16_e32:
3869 case AMDGPU::V_MAC_F16_e64:
3870 return AMDGPU::V_MAD_F16_e64;
3871 case AMDGPU::V_MAC_F32_e32:
3872 case AMDGPU::V_MAC_F32_e64:
3873 return AMDGPU::V_MAD_F32_e64;
3874 case AMDGPU::V_MAC_LEGACY_F32_e32:
3875 case AMDGPU::V_MAC_LEGACY_F32_e64:
3876 return AMDGPU::V_MAD_LEGACY_F32_e64;
3877 case AMDGPU::V_FMAC_LEGACY_F32_e32:
3878 case AMDGPU::V_FMAC_LEGACY_F32_e64:
3879 return AMDGPU::V_FMA_LEGACY_F32_e64;
3880 case AMDGPU::V_FMAC_F16_e32:
3881 case AMDGPU::V_FMAC_F16_e64:
3882 case AMDGPU::V_FMAC_F16_fake16_e64:
3883 return ST.hasTrue16BitInsts() ? AMDGPU::V_FMA_F16_gfx9_fake16_e64
3884 : AMDGPU::V_FMA_F16_gfx9_e64;
3885 case AMDGPU::V_FMAC_F32_e32:
3886 case AMDGPU::V_FMAC_F32_e64:
3887 return AMDGPU::V_FMA_F32_e64;
3888 case AMDGPU::V_FMAC_F64_e32:
3889 case AMDGPU::V_FMAC_F64_e64:
3890 return AMDGPU::V_FMA_F64_e64;
3903 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc);
3936 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode());
3950 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
3955 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 ||
3956 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3957 Opc == AMDGPU::V_FMAC_F16_fake16_e64;
3958 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3959 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3960 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 ||
3961 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3962 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
3963 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3964 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3965 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
3966 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
3967 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3968 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
3974 case AMDGPU::V_MAC_F16_e64:
3975 case AMDGPU::V_FMAC_F16_e64:
3976 case AMDGPU::V_FMAC_F16_fake16_e64:
3977 case AMDGPU::V_MAC_F32_e64:
3978 case AMDGPU::V_MAC_LEGACY_F32_e64:
3979 case AMDGPU::V_FMAC_F32_e64:
3980 case AMDGPU::V_FMAC_LEGACY_F32_e64:
3981 case AMDGPU::V_FMAC_F64_e64:
3983 case AMDGPU::V_MAC_F16_e32:
3984 case AMDGPU::V_FMAC_F16_e32:
3985 case AMDGPU::V_MAC_F32_e32:
3986 case AMDGPU::V_MAC_LEGACY_F32_e32:
3987 case AMDGPU::V_FMAC_F32_e32:
3988 case AMDGPU::V_FMAC_LEGACY_F32_e32:
3989 case AMDGPU::V_FMAC_F64_e32: {
3990 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3991 AMDGPU::OpName::src0);
4004 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4005 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
4007 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
4008 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
4010 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
4011 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
4013 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers);
4014 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
4015 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
4016 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel);
4031 DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF));
4061 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_fake16
4062 : AMDGPU::V_FMAAK_F16)
4063 : AMDGPU::V_FMAAK_F32)
4064 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
4080 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_fake16
4081 : AMDGPU::V_FMAMK_F16)
4082 : AMDGPU::V_FMAMK_F32)
4083 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
4107 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
4147 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel))
4160 case AMDGPU::S_SET_GPR_IDX_ON:
4161 case AMDGPU::S_SET_GPR_IDX_MODE:
4162 case AMDGPU::S_SET_GPR_IDX_OFF:
4187 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0)
4193 return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
4194 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4195 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4196 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4201 return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode);
4208 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE);
4227 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4228 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
4229 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT)
4248 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4249 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4250 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4251 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4268 return MI.readsRegister(AMDGPU::EXEC, &RI);
4279 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
4288 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
4291 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
4295 AMDGPU::isInlinableLiteralI16(Imm.getSExtValue(),
4314 AMDGPU::isInlinableLiteralBF16(IntImmVal, HasInv2Pi);
4317 AMDGPU::isInlinableLiteralFP16(IntImmVal, HasInv2Pi);
4334 case AMDGPU::OPERAND_REG_IMM_INT32:
4335 case AMDGPU::OPERAND_REG_IMM_FP32:
4336 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
4337 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4338 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4339 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4340 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
4341 case AMDGPU::OPERAND_REG_IMM_V2INT32:
4342 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
4343 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4344 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4345 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4347 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
4349 case AMDGPU::OPERAND_REG_IMM_INT64:
4350 case AMDGPU::OPERAND_REG_IMM_FP64:
4351 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4352 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4353 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4354 return AMDGPU::isInlinableLiteral64(MO.getImm(),
4356 case AMDGPU::OPERAND_REG_IMM_INT16:
4357 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4358 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
4369 return AMDGPU::isInlinableIntLiteral(Imm);
4370 case AMDGPU::OPERAND_REG_IMM_V2INT16:
4371 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4372 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
4373 return AMDGPU::isInlinableLiteralV2I16(Imm);
4374 case AMDGPU::OPERAND_REG_IMM_V2FP16:
4375 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4376 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
4377 return AMDGPU::isInlinableLiteralV2F16(Imm);
4378 case AMDGPU::OPERAND_REG_IMM_V2BF16:
4379 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
4380 case AMDGPU::OPERAND_REG_INLINE_AC_V2BF16:
4381 return AMDGPU::isInlinableLiteralV2BF16(Imm);
4382 case AMDGPU::OPERAND_REG_IMM_FP16:
4383 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
4384 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4385 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
4393 AMDGPU::isInlinableLiteralFP16(Trunc, ST.hasInv2PiInlineImm());
4398 case AMDGPU::OPERAND_REG_IMM_BF16:
4399 case AMDGPU::OPERAND_REG_IMM_BF16_DEFERRED:
4400 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
4401 case AMDGPU::OPERAND_REG_INLINE_AC_BF16: {
4405 AMDGPU::isInlinableLiteralBF16(Trunc, ST.hasInv2PiInlineImm());
4409 case AMDGPU::OPERAND_KIMM32:
4410 case AMDGPU::OPERAND_KIMM16:
4412 case AMDGPU::OPERAND_INPUT_MODS:
4462 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
4463 AMDGPU::OpName::src2))
4471 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
4479 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
4482 int Op32 = AMDGPU::getVOPe32(Opcode);
4493 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers);
4509 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
4515 case AMDGPU::V_ADDC_U32_e64:
4516 case AMDGPU::V_SUBB_U32_e64:
4517 case AMDGPU::V_SUBBREV_U32_e64: {
4519 = getNamedOperand(MI, AMDGPU::OpName::src1);
4525 case AMDGPU::V_MAC_F16_e64:
4526 case AMDGPU::V_MAC_F32_e64:
4527 case AMDGPU::V_MAC_LEGACY_F32_e64:
4528 case AMDGPU::V_FMAC_F16_e64:
4529 case AMDGPU::V_FMAC_F16_fake16_e64:
4530 case AMDGPU::V_FMAC_F32_e64:
4531 case AMDGPU::V_FMAC_F64_e64:
4532 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4534 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
4538 case AMDGPU::V_CNDMASK_B32_e64:
4543 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
4545 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
4550 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
4558 return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
4559 !hasModifiersSet(MI, AMDGPU::OpName::clamp) &&
4560 !hasModifiersSet(MI, AMDGPU::OpName::byte_sel) &&
4563 !hasModifiersSet(MI, AMDGPU::OpName::bound_ctrl) &&
4564 !hasModifiersSet(MI, AMDGPU::OpName::fi);
4574 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
4599 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
4604 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
4608 if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2) == -1) {
4641 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64)
4646 return MO.getReg() == AMDGPU::M0 || MO.getReg() == AMDGPU::VCC ||
4647 MO.getReg() == AMDGPU::VCC_LO;
4649 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
4650 AMDGPU::SReg_64RegClass.contains(MO.getReg());
4660 case AMDGPU::VCC:
4661 case AMDGPU::VCC_LO:
4662 case AMDGPU::VCC_HI:
4663 case AMDGPU::M0:
4664 case AMDGPU::FLAT_SCR:
4678 case AMDGPU::V_READLANE_B32:
4679 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
4680 case AMDGPU::V_WRITELANE_B32:
4681 case AMDGPU::SI_SPILL_S32_TO_VGPR:
4707 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
4743 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4744 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4745 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
4749 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);
4750 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);
4751 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);
4752 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);
4808 case AMDGPU::OPERAND_REG_IMM_INT32:
4809 case AMDGPU::OPERAND_REG_IMM_FP32:
4810 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
4811 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4813 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4814 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4815 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4816 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4817 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4818 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4819 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4820 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4821 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
4822 case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
4823 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
4830 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
4837 case AMDGPU::OPERAND_KIMM32:
4897 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4922 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4930 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
4931 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
4932 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
4933 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
4935 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
4944 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
4949 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
4955 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
4962 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4970 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
4972 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
4997 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
5002 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
5003 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
5004 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
5015 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
5030 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
5035 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
5087 Opcode != AMDGPU::V_WRITELANE_B32) {
5100 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
5111 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
5125 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
5126 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
5137 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
5139 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &
5141 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &
5162 const auto *Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
5184 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
5185 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
5186 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5187 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
5188 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5189 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
5203 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
5231 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
5242 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);
5243 if (Soff && Soff->getReg() != AMDGPU::M0) {
5251 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
5259 const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds);
5267 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
5269 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
5270 AMDGPU::OpName::vaddr0);
5272 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
5273 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName);
5274 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
5275 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
5276 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
5277 const AMDGPU::MIMGDimInfo *Dim =
5278 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm());
5287 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);
5290 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);
5297 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16());
5322 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
5324 using namespace AMDGPU::DPP;
5366 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
5367 !AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(Desc)) {
5375 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
5376 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0
5377 : AMDGPU::OpName::vdata;
5379 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);
5420 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
5421 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
5422 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
5424 if (!isAlignedReg(AMDGPU::OpName::data0)) {
5432 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
5440 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
5442 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);
5450 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
5466 default: return AMDGPU::INSTRUCTION_LIST_END;
5467 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
5468 case AMDGPU::COPY: return AMDGPU::COPY;
5469 case AMDGPU::PHI: return AMDGPU::PHI;
5470 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
5471 case AMDGPU::WQM: return AMDGPU::WQM;
5472 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
5473 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
5474 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
5475 case AMDGPU::S_MOV_B32: {
5479 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
5481 case AMDGPU::S_ADD_I32:
5482 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
5483 case AMDGPU::S_ADDC_U32:
5484 return AMDGPU::V_ADDC_U32_e32;
5485 case AMDGPU::S_SUB_I32:
5486 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
5489 case AMDGPU::S_ADD_U32:
5490 return AMDGPU::V_ADD_CO_U32_e32;
5491 case AMDGPU::S_SUB_U32:
5492 return AMDGPU::V_SUB_CO_U32_e32;
5493 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
5494 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
5495 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
5496 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
5497 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
5498 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
5499 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
5500 case AMDGPU::S_XNOR_B32:
5501 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
5502 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
5503 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
5504 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
5505 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
5506 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
5507 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
5508 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
5509 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
5510 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
5511 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
5512 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
5513 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
5514 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
5515 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
5516 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
5517 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
5518 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
5519 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
5520 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
5521 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
5522 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
5523 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
5524 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
5525 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
5526 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
5527 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
5528 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
5529 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
5530 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
5531 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
5532 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
5533 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
5534 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
5535 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
5536 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
5537 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
5538 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
5539 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
5540 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
5541 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
5542 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
5543 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
5544 case AMDGPU::S_CVT_F32_F16:
5545 case AMDGPU::S_CVT_HI_F32_F16:
5546 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
5547 : AMDGPU::V_CVT_F32_F16_fake16_e64;
5548 case AMDGPU::S_CVT_F16_F32:
5549 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
5550 : AMDGPU::V_CVT_F16_F32_fake16_e64;
5551 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
5552 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
5553 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
5554 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
5555 case AMDGPU::S_CEIL_F16:
5556 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
5557 : AMDGPU::V_CEIL_F16_fake16_e64;
5558 case AMDGPU::S_FLOOR_F16:
5559 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
5560 : AMDGPU::V_FLOOR_F16_fake16_e64;
5561 case AMDGPU::S_TRUNC_F16:
5562 return AMDGPU::V_TRUNC_F16_fake16_e64;
5563 case AMDGPU::S_RNDNE_F16:
5564 return AMDGPU::V_RNDNE_F16_fake16_e64;
5565 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
5566 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
5567 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
5568 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
5569 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
5570 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
5571 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
5572 case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64;
5573 case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64;
5574 case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64;
5575 case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64;
5576 case AMDGPU::S_MINIMUM_F16:
5577 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
5578 : AMDGPU::V_MINIMUM_F16_fake16_e64;
5579 case AMDGPU::S_MAXIMUM_F16:
5580 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
5581 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
5582 case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64;
5583 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
5584 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
5585 case AMDGPU::S_FMAC_F16: return AMDGPU::V_FMAC_F16_fake16_e64;
5586 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
5587 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
5588 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
5589 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
5590 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
5591 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
5592 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
5593 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
5594 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
5595 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
5596 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
5597 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
5598 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
5599 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
5600 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
5601 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
5602 case AMDGPU::S_CMP_LT_F16:
5603 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
5604 : AMDGPU::V_CMP_LT_F16_fake16_e64;
5605 case AMDGPU::S_CMP_EQ_F16:
5606 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
5607 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
5608 case AMDGPU::S_CMP_LE_F16:
5609 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
5610 : AMDGPU::V_CMP_LE_F16_fake16_e64;
5611 case AMDGPU::S_CMP_GT_F16:
5612 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
5613 : AMDGPU::V_CMP_GT_F16_fake16_e64;
5614 case AMDGPU::S_CMP_LG_F16:
5615 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
5616 : AMDGPU::V_CMP_LG_F16_fake16_e64;
5617 case AMDGPU::S_CMP_GE_F16:
5618 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
5619 : AMDGPU::V_CMP_GE_F16_fake16_e64;
5620 case AMDGPU::S_CMP_O_F16:
5621 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
5622 : AMDGPU::V_CMP_O_F16_fake16_e64;
5623 case AMDGPU::S_CMP_U_F16:
5624 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
5625 : AMDGPU::V_CMP_U_F16_fake16_e64;
5626 case AMDGPU::S_CMP_NGE_F16:
5627 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
5628 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
5629 case AMDGPU::S_CMP_NLG_F16:
5630 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
5631 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
5632 case AMDGPU::S_CMP_NGT_F16:
5633 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
5634 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
5635 case AMDGPU::S_CMP_NLE_F16:
5636 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
5637 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
5638 case AMDGPU::S_CMP_NEQ_F16:
5639 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
5640 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
5641 case AMDGPU::S_CMP_NLT_F16:
5642 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
5643 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
5644 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
5645 case AMDGPU::V_S_EXP_F16_e64: return AMDGPU::V_EXP_F16_fake16_e64;
5646 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
5647 case AMDGPU::V_S_LOG_F16_e64: return AMDGPU::V_LOG_F16_fake16_e64;
5648 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
5649 case AMDGPU::V_S_RCP_F16_e64: return AMDGPU::V_RCP_F16_fake16_e64;
5650 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
5651 case AMDGPU::V_S_RSQ_F16_e64: return AMDGPU::V_RSQ_F16_fake16_e64;
5652 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
5653 case AMDGPU::V_S_SQRT_F16_e64: return AMDGPU::V_SQRT_F16_fake16_e64;
5674 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5675 MCRegister Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5685 IsWave32 ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64;
5698 unsigned ExecMov = isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5699 MCRegister Exec = isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5716 case AMDGPU::AV_32RegClassID:
5717 RCID = AMDGPU::VGPR_32RegClassID;
5719 case AMDGPU::AV_64RegClassID:
5720 RCID = AMDGPU::VReg_64RegClassID;
5722 case AMDGPU::AV_96RegClassID:
5723 RCID = AMDGPU::VReg_96RegClassID;
5725 case AMDGPU::AV_128RegClassID:
5726 RCID = AMDGPU::VReg_128RegClassID;
5728 case AMDGPU::AV_160RegClassID:
5729 RCID = AMDGPU::VReg_160RegClassID;
5731 case AMDGPU::AV_512RegClassID:
5732 RCID = AMDGPU::VReg_512RegClassID;
5758 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
5759 AMDGPU::OpName::vdst);
5760 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
5761 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
5762 : AMDGPU::OpName::vdata);
5764 IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand(
5765 TID.Opcode, AMDGPU::OpName::data1);
5797 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
5798 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
5799 : AMDGPU::V_MOV_B32_e32;
5801 Opcode = AMDGPU::COPY;
5803 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
5834 if (SubIdx == AMDGPU::sub0)
5836 if (SubIdx == AMDGPU::sub1)
5899 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
5900 const int DataIdx = AMDGPU::getNamedOperandIdx(
5901 Opc, isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
5911 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
5918 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
5919 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) &&
5970 } else if (AMDGPU::isSISrcOperand(InstDesc, i) &&
5992 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64;
5994 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
5995 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
5996 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
5998 !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm())) {
5999 if (!AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp))
6027 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
6030 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
6043 if (Opc == AMDGPU::V_WRITELANE_B32) {
6046 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6047 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6052 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6054 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6069 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
6070 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
6083 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
6085 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6087 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6145 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
6146 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
6147 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
6150 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
6151 Opc == AMDGPU::V_PERMLANEX16_B32_e64) {
6157 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6158 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6163 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6164 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6226 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
6253 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
6260 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6262 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
6269 get(AMDGPU::REG_SEQUENCE), DstReg);
6284 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
6289 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset);
6298 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
6304 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
6306 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc);
6315 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
6319 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
6326 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 ||
6351 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc,
6352 AMDGPU::OpName::vdst_in);
6357 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in);
6364 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);
6384 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr);
6416 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).addReg(OpReg);
6424 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
6434 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
6436 Copy.addReg(AMDGPU::EXEC, RegState::Implicit);
6452 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6454 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
6456 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
6458 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6470 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6472 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
6477 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
6502 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6503 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6506 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
6510 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
6518 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
6519 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
6521 .addImm(AMDGPU::sub0)
6523 .addImm(AMDGPU::sub1);
6526 auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
6553 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
6580 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
6604 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6605 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6614 MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI,
6618 SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6619 BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg)
6682 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32))
6701 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
6702 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
6705 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6706 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6707 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6708 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
6712 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
6716 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
6720 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
6724 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
6726 .addImm(AMDGPU::sub0_sub1)
6728 .addImm(AMDGPU::sub2)
6730 .addImm(AMDGPU::sub3);
6769 if (MI.getOpcode() == AMDGPU::PHI) {
6789 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) {
6790 VRC = &AMDGPU::VReg_1RegClass;
6824 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
6851 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
6865 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
6873 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
6874 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
6875 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
6876 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
6877 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
6878 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
6879 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
6891 if (isImage(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) &&
6893 int RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI)) ? AMDGPU::OpName::rsrc
6894 : AMDGPU::OpName::srsrc;
6899 int SampOpName = isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
6908 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
6936 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
6938 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6940 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
6942 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6951 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset);
6962 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
6990 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
6991 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
6994 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6995 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6996 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
7007 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo)
7009 .addReg(RsrcPtr, 0, AMDGPU::sub0)
7010 .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
7014 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
7016 .addReg(RsrcPtr, 0, AMDGPU::sub1)
7017 .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
7022 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
7024 .addImm(AMDGPU::sub0)
7026 .addImm(AMDGPU::sub1);
7039 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
7040 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
7041 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
7042 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
7043 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
7047 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
7061 getNamedOperand(MI, AMDGPU::OpName::cpol)) {
7066 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
7070 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
7083 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol))
7090 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
7092 .addReg(RsrcPtr, 0, AMDGPU::sub0)
7093 .addImm(AMDGPU::sub0)
7094 .addReg(RsrcPtr, 0, AMDGPU::sub1)
7095 .addImm(AMDGPU::sub1);
7099 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
7111 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
7122 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc);
7167 case AMDGPU::S_ADD_U64_PSEUDO:
7168 NewOpcode = AMDGPU::V_ADD_U64_PSEUDO;
7170 case AMDGPU::S_SUB_U64_PSEUDO:
7171 NewOpcode = AMDGPU::V_SUB_U64_PSEUDO;
7173 case AMDGPU::S_ADD_I32:
7174 case AMDGPU::S_SUB_I32: {
7186 case AMDGPU::S_MUL_U64:
7192 case AMDGPU::S_MUL_U64_U32_PSEUDO:
7193 case AMDGPU::S_MUL_I64_I32_PSEUDO:
7200 case AMDGPU::S_AND_B64:
7201 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
7205 case AMDGPU::S_OR_B64:
7206 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
7210 case AMDGPU::S_XOR_B64:
7211 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
7215 case AMDGPU::S_NAND_B64:
7216 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
7220 case AMDGPU::S_NOR_B64:
7221 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
7225 case AMDGPU::S_XNOR_B64:
7227 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
7233 case AMDGPU::S_ANDN2_B64:
7234 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
7238 case AMDGPU::S_ORN2_B64:
7239 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
7243 case AMDGPU::S_BREV_B64:
7244 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true);
7248 case AMDGPU::S_NOT_B64:
7249 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
7253 case AMDGPU::S_BCNT1_I32_B64:
7258 case AMDGPU::S_BFE_I64:
7263 case AMDGPU::S_FLBIT_I32_B64:
7264 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32);
7267 case AMDGPU::S_FF1_I32_B64:
7268 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32);
7272 case AMDGPU::S_LSHL_B32:
7274 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
7278 case AMDGPU::S_ASHR_I32:
7280 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
7284 case AMDGPU::S_LSHR_B32:
7286 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
7290 case AMDGPU::S_LSHL_B64:
7293 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
7294 : AMDGPU::V_LSHLREV_B64_e64;
7298 case AMDGPU::S_ASHR_I64:
7300 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
7304 case AMDGPU::S_LSHR_B64:
7306 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
7311 case AMDGPU::S_ABS_I32:
7316 case AMDGPU::S_CBRANCH_SCC0:
7317 case AMDGPU::S_CBRANCH_SCC1: {
7320 bool IsSCC = CondReg == AMDGPU::SCC;
7322 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
7323 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
7330 case AMDGPU::S_BFE_U64:
7331 case AMDGPU::S_BFM_B64:
7334 case AMDGPU::S_PACK_LL_B32_B16:
7335 case AMDGPU::S_PACK_LH_B32_B16:
7336 case AMDGPU::S_PACK_HL_B32_B16:
7337 case AMDGPU::S_PACK_HH_B32_B16:
7342 case AMDGPU::S_XNOR_B32:
7347 case AMDGPU::S_NAND_B32:
7348 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
7352 case AMDGPU::S_NOR_B32:
7353 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
7357 case AMDGPU::S_ANDN2_B32:
7358 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
7362 case AMDGPU::S_ORN2_B32:
7363 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
7371 case AMDGPU::S_ADD_CO_PSEUDO:
7372 case AMDGPU::S_SUB_CO_PSEUDO: {
7373 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
7374 ? AMDGPU::V_ADDC_U32_e64
7375 : AMDGPU::V_SUBB_U32_e64;
7381 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg)
7402 case AMDGPU::S_UADDO_PSEUDO:
7403 case AMDGPU::S_USUBO_PSEUDO: {
7410 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7411 ? AMDGPU::V_ADD_CO_U32_e64
7412 : AMDGPU::V_SUB_CO_U32_e64;
7430 case AMDGPU::S_CSELECT_B32:
7431 case AMDGPU::S_CSELECT_B64:
7435 case AMDGPU::S_CMP_EQ_I32:
7436 case AMDGPU::S_CMP_LG_I32:
7437 case AMDGPU::S_CMP_GT_I32:
7438 case AMDGPU::S_CMP_GE_I32:
7439 case AMDGPU::S_CMP_LT_I32:
7440 case AMDGPU::S_CMP_LE_I32:
7441 case AMDGPU::S_CMP_EQ_U32:
7442 case AMDGPU::S_CMP_LG_U32:
7443 case AMDGPU::S_CMP_GT_U32:
7444 case AMDGPU::S_CMP_GE_U32:
7445 case AMDGPU::S_CMP_LT_U32:
7446 case AMDGPU::S_CMP_LE_U32:
7447 case AMDGPU::S_CMP_EQ_U64:
7448 case AMDGPU::S_CMP_LG_U64:
7449 case AMDGPU::S_CMP_LT_F32:
7450 case AMDGPU::S_CMP_EQ_F32:
7451 case AMDGPU::S_CMP_LE_F32:
7452 case AMDGPU::S_CMP_GT_F32:
7453 case AMDGPU::S_CMP_LG_F32:
7454 case AMDGPU::S_CMP_GE_F32:
7455 case AMDGPU::S_CMP_O_F32:
7456 case AMDGPU::S_CMP_U_F32:
7457 case AMDGPU::S_CMP_NGE_F32:
7458 case AMDGPU::S_CMP_NLG_F32:
7459 case AMDGPU::S_CMP_NGT_F32:
7460 case AMDGPU::S_CMP_NLE_F32:
7461 case AMDGPU::S_CMP_NEQ_F32:
7462 case AMDGPU::S_CMP_NLT_F32: {
7467 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src0_modifiers) >=
7479 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC, /*TRI=*/nullptr);
7485 case AMDGPU::S_CMP_LT_F16:
7486 case AMDGPU::S_CMP_EQ_F16:
7487 case AMDGPU::S_CMP_LE_F16:
7488 case AMDGPU::S_CMP_GT_F16:
7489 case AMDGPU::S_CMP_LG_F16:
7490 case AMDGPU::S_CMP_GE_F16:
7491 case AMDGPU::S_CMP_O_F16:
7492 case AMDGPU::S_CMP_U_F16:
7493 case AMDGPU::S_CMP_NGE_F16:
7494 case AMDGPU::S_CMP_NLG_F16:
7495 case AMDGPU::S_CMP_NGT_F16:
7496 case AMDGPU::S_CMP_NLE_F16:
7497 case AMDGPU::S_CMP_NEQ_F16:
7498 case AMDGPU::S_CMP_NLT_F16: {
7503 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0_modifiers)) {
7510 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::op_sel))
7518 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC, /*TRI=*/nullptr);
7524 case AMDGPU::S_CVT_HI_F32_F16: {
7526 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7527 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7529 BuildMI(*MBB, Inst, DL, get(AMDGPU::COPY), TmpReg)
7533 .addReg(TmpReg, 0, AMDGPU::hi16)
7538 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
7553 case AMDGPU::S_MINIMUM_F32:
7554 case AMDGPU::S_MAXIMUM_F32: {
7556 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7571 case AMDGPU::S_MINIMUM_F16:
7572 case AMDGPU::S_MAXIMUM_F16: {
7575 ? &AMDGPU::VGPR_16RegClass
7576 : &AMDGPU::VGPR_32RegClass);
7593 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
7611 get(AMDGPU::V_READFIRSTLANE_B32), Inst.getOperand(0).getReg())
7634 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
7650 if (AMDGPU::getNamedOperandIdx(NewOpcode,
7651 AMDGPU::OpName::src0_modifiers) >= 0)
7653 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0)) {
7655 if (AMDGPU::isTrue16Inst(NewOpcode) && ST.useRealTrue16Insts() &&
7657 NewInstr.addReg(Src.getReg(), 0, AMDGPU::lo16);
7662 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
7665 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
7668 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
7672 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
7685 if (AMDGPU::getNamedOperandIdx(NewOpcode,
7686 AMDGPU::OpName::src1_modifiers) >= 0)
7688 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0)
7690 if (AMDGPU::getNamedOperandIdx(NewOpcode,
7691 AMDGPU::OpName::src2_modifiers) >= 0)
7693 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0)
7695 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0)
7697 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0)
7699 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0)
7712 if (Op.getReg() == AMDGPU::SCC) {
7751 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7754 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
7756 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
7757 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
7759 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
7789 bool IsSCC = (CondReg == AMDGPU::SCC);
7812 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI, false, false) !=
7814 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {
7815 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg)
7828 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7838 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
7839 NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg)
7847 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg)
7866 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7867 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7870 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32;
7876 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
7896 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7897 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
7898 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
7900 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
7916 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7917 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7923 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
7924 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
7928 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
7929 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
7933 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
7937 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
7961 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7962 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7968 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
7990 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7991 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7993 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
8022 &AMDGPU::SGPR_32RegClass;
8025 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
8028 AMDGPU::sub0, Src0SubRC);
8033 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
8039 AMDGPU::sub1, Src0SubRC);
8050 .addImm(AMDGPU::sub0)
8052 .addImm(AMDGPU::sub1);
8074 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8075 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8076 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8087 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
8091 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
8098 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
8100 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
8102 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
8104 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
8121 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8123 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg)
8127 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8129 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg)
8133 Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8135 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg)
8140 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)
8144 Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8145 MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg)
8150 BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1)
8156 .addImm(AMDGPU::sub0)
8158 .addImm(AMDGPU::sub1);
8183 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8184 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8185 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8196 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
8200 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
8207 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
8209 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
8212 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
8213 ? AMDGPU::V_MUL_HI_U32_e64
8214 : AMDGPU::V_MUL_HI_I32_e64;
8219 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)
8225 .addImm(AMDGPU::sub0)
8227 .addImm(AMDGPU::sub1);
8256 &AMDGPU::SGPR_32RegClass;
8259 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
8262 &AMDGPU::SGPR_32RegClass;
8265 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
8268 AMDGPU::sub0, Src0SubRC);
8270 AMDGPU::sub0, Src1SubRC);
8272 AMDGPU::sub1, Src0SubRC);
8274 AMDGPU::sub1, Src1SubRC);
8279 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
8294 .addImm(AMDGPU::sub0)
8296 .addImm(AMDGPU::sub1);
8322 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
8335 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
8340 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
8360 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
8363 &AMDGPU::SGPR_32RegClass;
8365 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8366 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8369 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
8372 AMDGPU::sub0, SrcSubRC);
8374 AMDGPU::sub1, SrcSubRC);
8402 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
8406 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8407 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8408 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8410 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo)
8411 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
8415 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
8421 .addImm(AMDGPU::sub0)
8423 .addImm(AMDGPU::sub1);
8431 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8432 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8434 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
8436 .addReg(Src.getReg(), 0, AMDGPU::sub0);
8439 .addReg(Src.getReg(), 0, AMDGPU::sub0)
8440 .addImm(AMDGPU::sub0)
8442 .addImm(AMDGPU::sub1);
8466 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
8468 ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
8471 Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;
8473 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
8476 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC);
8478 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC);
8480 Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8481 Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8482 Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8483 Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8494 BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4)
8513 case AMDGPU::COPY:
8514 case AMDGPU::WQM:
8515 case AMDGPU::SOFT_WQM:
8516 case AMDGPU::STRICT_WWM:
8517 case AMDGPU::STRICT_WQM:
8518 case AMDGPU::REG_SEQUENCE:
8519 case AMDGPU::PHI:
8520 case AMDGPU::INSERT_SUBREG:
8542 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8549 case AMDGPU::S_PACK_LL_B32_B16: {
8550 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8551 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8555 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
8558 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
8562 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)
8568 case AMDGPU::S_PACK_LH_B32_B16: {
8569 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8570 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
8572 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg)
8578 case AMDGPU::S_PACK_HL_B32_B16: {
8579 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8580 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
8583 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)
8589 case AMDGPU::S_PACK_HH_B32_B16: {
8590 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8591 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8592 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
8595 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
8597 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg)
8618 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
8627 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, &RI, false);
8644 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, &RI, false, false) != -1)
8665 if (MI.modifiesRegister(AMDGPU::VCC, &RI))
8667 if (MI.definesRegister(AMDGPU::SCC, &RI)) {
8682 case AMDGPU::COPY:
8683 case AMDGPU::PHI:
8684 case AMDGPU::REG_SEQUENCE:
8685 case AMDGPU::INSERT_SUBREG:
8686 case AMDGPU::WQM:
8687 case AMDGPU::SOFT_WQM:
8688 case AMDGPU::STRICT_WWM:
8689 case AMDGPU::STRICT_WQM: {
8696 case AMDGPU::PHI:
8697 case AMDGPU::REG_SEQUENCE:
8698 case AMDGPU::INSERT_SUBREG:
8708 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
8796 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
8806 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
8807 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
8813 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
8830 AMDGPU::RSRC_TID_ENABLE |
8836 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
8841 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
8847 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
8865 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
8873 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
8878 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
8881 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
8961 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
8965 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
9002 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
9003 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
9004 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
9005 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
9006 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
9074 if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG))
9075 return AMDGPU::WWM_COPY;
9077 return AMDGPU::COPY;
9099 (Opcode == AMDGPU::IMPLICIT_DEF &&
9101 (!MI.isTerminator() && Opcode != AMDGPU::COPY &&
9102 MI.modifiesRegister(AMDGPU::EXEC, &RI)));
9111 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
9117 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)
9127 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg);
9130 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC)
9140 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)
9146 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
9147 case AMDGPU::SI_KILL_I1_TERMINATOR:
9156 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
9157 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
9158 case AMDGPU::SI_KILL_I1_PSEUDO:
9159 return get(AMDGPU::SI_KILL_I1_TERMINATOR);
9184 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
9185 Op.setReg(AMDGPU::VCC_LO);
9194 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
9199 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass);
9303 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
9315 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
9345 return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(ST);
9374 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
9375 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
9376 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
9377 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
9378 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
9379 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
9380 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
9381 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
9397 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
9398 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
9399 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
9400 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
9401 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
9402 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
9403 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
9404 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
9405 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
9407 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
9408 case AMDGPU::V_DIV_FIXUP_F16_gfx9_fake16_e64:
9409 case AMDGPU::V_FMA_F16_gfx9_e64:
9410 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:
9411 case AMDGPU::V_INTERP_P2_F16:
9412 case AMDGPU::V_MAD_F16_e64:
9413 case AMDGPU::V_MAD_U16_e64:
9414 case AMDGPU::V_MAD_I16_e64:
9450 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
9455 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
9464 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940);
9466 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A);
9468 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9);
9510 case AMDGPU::REG_SEQUENCE:
9514 case AMDGPU::INSERT_SUBREG:
9540 case AMDGPU::COPY:
9541 case AMDGPU::V_MOV_B32_e32: {
9590 if (I->modifiesRegister(AMDGPU::EXEC, TRI))
9647 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC))
9672 (InsPt->getOpcode() == AMDGPU::SI_IF ||
9673 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
9674 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
9678 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term
9679 : AMDGPU::S_MOV_B64_term),
9682 .addReg(AMDGPU::EXEC, RegState::Implicit);
9714 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) {
9715 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
9718 if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) {
9719 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass);
9750 if (AMDGPU::isIntrinsicSourceOfDivergence(IID))
9752 if (AMDGPU::isIntrinsicAlwaysUniform(IID))
9771 if (opcode == AMDGPU::G_LOAD) {
9786 opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
9787 opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
9788 AMDGPU::isGenericAtomic(opcode)) {
9801 if (opcode == AMDGPU::V_READLANE_B32 ||
9802 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
9803 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
9865 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
9910 case AMDGPU::S_CMP_EQ_U32:
9911 case AMDGPU::S_CMP_EQ_I32:
9912 case AMDGPU::S_CMP_LG_U32:
9913 case AMDGPU::S_CMP_LG_I32:
9914 case AMDGPU::S_CMP_LT_U32:
9915 case AMDGPU::S_CMP_LT_I32:
9916 case AMDGPU::S_CMP_GT_U32:
9917 case AMDGPU::S_CMP_GT_I32:
9918 case AMDGPU::S_CMP_LE_U32:
9919 case AMDGPU::S_CMP_LE_I32:
9920 case AMDGPU::S_CMP_GE_U32:
9921 case AMDGPU::S_CMP_GE_I32:
9922 case AMDGPU::S_CMP_EQ_U64:
9923 case AMDGPU::S_CMP_LG_U64:
9938 case AMDGPU::S_CMPK_EQ_U32:
9939 case AMDGPU::S_CMPK_EQ_I32:
9940 case AMDGPU::S_CMPK_LG_U32:
9941 case AMDGPU::S_CMPK_LG_I32:
9942 case AMDGPU::S_CMPK_LT_U32:
9943 case AMDGPU::S_CMPK_LT_I32:
9944 case AMDGPU::S_CMPK_GT_U32:
9945 case AMDGPU::S_CMPK_GT_I32:
9946 case AMDGPU::S_CMPK_LE_U32:
9947 case AMDGPU::S_CMPK_LE_I32:
9948 case AMDGPU::S_CMPK_GE_U32:
9949 case AMDGPU::S_CMPK_GE_I32:
10000 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
10001 Def->getOpcode() != AMDGPU::S_AND_B64)
10046 if (I->modifiesRegister(AMDGPU::SCC, &RI) ||
10047 I->killsRegister(AMDGPU::SCC, &RI))
10052 Def->findRegisterDefOperand(AMDGPU::SCC, /*TRI=*/nullptr);
10064 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
10065 : AMDGPU::S_BITCMP1_B32
10066 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
10067 : AMDGPU::S_BITCMP1_B64;
10080 case AMDGPU::S_CMP_EQ_U32:
10081 case AMDGPU::S_CMP_EQ_I32:
10082 case AMDGPU::S_CMPK_EQ_U32:
10083 case AMDGPU::S_CMPK_EQ_I32:
10085 case AMDGPU::S_CMP_GE_U32:
10086 case AMDGPU::S_CMPK_GE_U32:
10088 case AMDGPU::S_CMP_GE_I32:
10089 case AMDGPU::S_CMPK_GE_I32:
10091 case AMDGPU::S_CMP_EQ_U64:
10093 case AMDGPU::S_CMP_LG_U32:
10094 case AMDGPU::S_CMP_LG_I32:
10095 case AMDGPU::S_CMPK_LG_U32:
10096 case AMDGPU::S_CMPK_LG_I32:
10098 case AMDGPU::S_CMP_GT_U32:
10099 case AMDGPU::S_CMPK_GT_U32:
10101 case AMDGPU::S_CMP_GT_I32:
10102 case AMDGPU::S_CMPK_GT_I32:
10104 case AMDGPU::S_CMP_LG_U64:
10116 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
10130 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
10131 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
10133 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
10134 : &AMDGPU::VReg_64_Align2RegClass);
10135 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR)
10137 .addImm(AMDGPU::sub0)
10139 .addImm(AMDGPU::sub1);
10141 Op.setSubReg(AMDGPU::sub0);