xref: /freebsd-src/contrib/llvm-project/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp (revision 1db9f3b21e39176dd5b67cf8ac378633b172463e)
1 //=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9 // operand. If any of the use instruction cannot be combined with the mov the
10 // whole sequence is reverted.
11 //
12 // $old = ...
13 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14 //                            dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
15 // $res = VALU $dpp_value [, src1]
16 //
17 // to
18 //
19 // $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
20 //                 dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
21 //
22 // Combining rules :
23 //
24 // if $row_mask and $bank_mask are fully enabled (0xF) and
25 //    $bound_ctrl==DPP_BOUND_ZERO or $old==0
26 // -> $combined_old = undef,
27 //    $combined_bound_ctrl = DPP_BOUND_ZERO
28 //
29 // if the VALU op is binary and
30 //    $bound_ctrl==DPP_BOUND_OFF and
31 //    $old==identity value (immediate) for the VALU op
32 // -> $combined_old = src1,
33 //    $combined_bound_ctrl = DPP_BOUND_OFF
34 //
35 // Otherwise cancel.
36 //
37 // The mov_dpp instruction should reside in the same BB as all its uses
38 //===----------------------------------------------------------------------===//
39 
40 #include "AMDGPU.h"
41 #include "GCNSubtarget.h"
42 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
43 #include "llvm/ADT/Statistic.h"
44 #include "llvm/CodeGen/MachineFunctionPass.h"
45 
46 using namespace llvm;
47 
48 #define DEBUG_TYPE "gcn-dpp-combine"
49 
50 STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
51 
52 namespace {
53 
54 class GCNDPPCombine : public MachineFunctionPass {
55   MachineRegisterInfo *MRI;
56   const SIInstrInfo *TII;
57   const GCNSubtarget *ST;
58 
59   using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
60 
61   MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
62 
63   MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
64                               RegSubRegPair CombOldVGPR,
65                               MachineOperand *OldOpnd, bool CombBCZ,
66                               bool IsShrinkable) const;
67 
68   MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
69                               RegSubRegPair CombOldVGPR, bool CombBCZ,
70                               bool IsShrinkable) const;
71 
72   bool hasNoImmOrEqual(MachineInstr &MI,
73                        unsigned OpndName,
74                        int64_t Value,
75                        int64_t Mask = -1) const;
76 
77   bool combineDPPMov(MachineInstr &MI) const;
78 
79 public:
80   static char ID;
81 
82   GCNDPPCombine() : MachineFunctionPass(ID) {
83     initializeGCNDPPCombinePass(*PassRegistry::getPassRegistry());
84   }
85 
86   bool runOnMachineFunction(MachineFunction &MF) override;
87 
88   StringRef getPassName() const override { return "GCN DPP Combine"; }
89 
90   void getAnalysisUsage(AnalysisUsage &AU) const override {
91     AU.setPreservesCFG();
92     MachineFunctionPass::getAnalysisUsage(AU);
93   }
94 
95   MachineFunctionProperties getRequiredProperties() const override {
96     return MachineFunctionProperties()
97       .set(MachineFunctionProperties::Property::IsSSA);
98   }
99 
100 private:
101   int getDPPOp(unsigned Op, bool IsShrinkable) const;
102   bool isShrinkable(MachineInstr &MI) const;
103 };
104 
105 } // end anonymous namespace
106 
107 INITIALIZE_PASS(GCNDPPCombine, DEBUG_TYPE, "GCN DPP Combine", false, false)
108 
109 char GCNDPPCombine::ID = 0;
110 
111 char &llvm::GCNDPPCombineID = GCNDPPCombine::ID;
112 
113 FunctionPass *llvm::createGCNDPPCombinePass() {
114   return new GCNDPPCombine();
115 }
116 
117 bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const {
118   unsigned Op = MI.getOpcode();
119   if (!TII->isVOP3(Op)) {
120     return false;
121   }
122   if (!TII->hasVALU32BitEncoding(Op)) {
123     LLVM_DEBUG(dbgs() << "  Inst hasn't e32 equivalent\n");
124     return false;
125   }
126   // Do not shrink True16 instructions pre-RA to avoid the restriction in
127   // register allocation from only being able to use 128 VGPRs
128   if (AMDGPU::isTrue16Inst(Op))
129     return false;
130   if (const auto *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) {
131     // Give up if there are any uses of the sdst in carry-out or VOPC.
132     // The shrunken form of the instruction would write it to vcc instead of to
133     // a virtual register. If we rewrote the uses the shrinking would be
134     // possible.
135     if (!MRI->use_nodbg_empty(SDst->getReg()))
136       return false;
137   }
138   // check if other than abs|neg modifiers are set (opsel for example)
139   const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
140   if (!hasNoImmOrEqual(MI, AMDGPU::OpName::src0_modifiers, 0, Mask) ||
141       !hasNoImmOrEqual(MI, AMDGPU::OpName::src1_modifiers, 0, Mask) ||
142       !hasNoImmOrEqual(MI, AMDGPU::OpName::clamp, 0) ||
143       !hasNoImmOrEqual(MI, AMDGPU::OpName::omod, 0)) {
144     LLVM_DEBUG(dbgs() << "  Inst has non-default modifiers\n");
145     return false;
146   }
147   return true;
148 }
149 
150 int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
151   int DPP32 = AMDGPU::getDPPOp32(Op);
152   if (IsShrinkable) {
153     assert(DPP32 == -1);
154     int E32 = AMDGPU::getVOPe32(Op);
155     DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(E32);
156   }
157   if (DPP32 != -1 && TII->pseudoToMCOpcode(DPP32) != -1)
158     return DPP32;
159   int DPP64 = -1;
160   if (ST->hasVOP3DPP())
161     DPP64 = AMDGPU::getDPPOp64(Op);
162   if (DPP64 != -1 && TII->pseudoToMCOpcode(DPP64) != -1)
163     return DPP64;
164   return -1;
165 }
166 
167 // tracks the register operand definition and returns:
168 //   1. immediate operand used to initialize the register if found
169 //   2. nullptr if the register operand is undef
170 //   3. the operand itself otherwise
171 MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
172   auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI);
173   if (!Def)
174     return nullptr;
175 
176   switch(Def->getOpcode()) {
177   default: break;
178   case AMDGPU::IMPLICIT_DEF:
179     return nullptr;
180   case AMDGPU::COPY:
181   case AMDGPU::V_MOV_B32_e32:
182   case AMDGPU::V_MOV_B64_PSEUDO:
183   case AMDGPU::V_MOV_B64_e32:
184   case AMDGPU::V_MOV_B64_e64: {
185     auto &Op1 = Def->getOperand(1);
186     if (Op1.isImm())
187       return &Op1;
188     break;
189   }
190   }
191   return &OldOpnd;
192 }
193 
194 [[maybe_unused]] static unsigned getOperandSize(MachineInstr &MI, unsigned Idx,
195                                MachineRegisterInfo &MRI) {
196   int16_t RegClass = MI.getDesc().operands()[Idx].RegClass;
197   if (RegClass == -1)
198     return 0;
199 
200   const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
201   return TRI->getRegSizeInBits(*TRI->getRegClass(RegClass));
202 }
203 
204 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
205                                            MachineInstr &MovMI,
206                                            RegSubRegPair CombOldVGPR,
207                                            bool CombBCZ,
208                                            bool IsShrinkable) const {
209   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
210          MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
211          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
212 
213   bool HasVOP3DPP = ST->hasVOP3DPP();
214   auto OrigOp = OrigMI.getOpcode();
215   auto DPPOp = getDPPOp(OrigOp, IsShrinkable);
216   if (DPPOp == -1) {
217     LLVM_DEBUG(dbgs() << "  failed: no DPP opcode\n");
218     return nullptr;
219   }
220   int OrigOpE32 = AMDGPU::getVOPe32(OrigOp);
221   // Prior checks cover Mask with VOPC condition, but not on purpose
222   auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
223   assert(RowMaskOpnd && RowMaskOpnd->isImm());
224   auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
225   assert(BankMaskOpnd && BankMaskOpnd->isImm());
226   const bool MaskAllLanes =
227       RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF;
228   (void)MaskAllLanes;
229   assert((MaskAllLanes ||
230           !(TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
231                                    TII->isVOPC(OrigOpE32)))) &&
232          "VOPC cannot form DPP unless mask is full");
233 
234   auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI,
235                          OrigMI.getDebugLoc(), TII->get(DPPOp))
236     .setMIFlags(OrigMI.getFlags());
237 
238   bool Fail = false;
239   do {
240     int NumOperands = 0;
241     if (auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst)) {
242       DPPInst.add(*Dst);
243       ++NumOperands;
244     }
245     if (auto *SDst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::sdst)) {
246       if (TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, SDst)) {
247         DPPInst.add(*SDst);
248         ++NumOperands;
249       }
250       // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst
251     }
252 
253     const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old);
254     if (OldIdx != -1) {
255       assert(OldIdx == NumOperands);
256       assert(isOfRegClass(
257           CombOldVGPR,
258           *MRI->getRegClass(
259               TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),
260           *MRI));
261       auto *Def = getVRegSubRegDef(CombOldVGPR, *MRI);
262       DPPInst.addReg(CombOldVGPR.Reg, Def ? 0 : RegState::Undef,
263                      CombOldVGPR.SubReg);
264       ++NumOperands;
265     } else if (TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
266                                       TII->isVOPC(OrigOpE32))) {
267       // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand
268       // because they write to SGPRs not VGPRs
269     } else {
270       // TODO: this discards MAC/FMA instructions for now, let's add it later
271       LLVM_DEBUG(dbgs() << "  failed: no old operand in DPP instruction,"
272                            " TBD\n");
273       Fail = true;
274       break;
275     }
276 
277     if (auto *Mod0 = TII->getNamedOperand(OrigMI,
278                                           AMDGPU::OpName::src0_modifiers)) {
279       assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
280                                           AMDGPU::OpName::src0_modifiers));
281       assert(HasVOP3DPP ||
282              (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
283       DPPInst.addImm(Mod0->getImm());
284       ++NumOperands;
285     } else if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src0_modifiers)) {
286       DPPInst.addImm(0);
287       ++NumOperands;
288     }
289     auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
290     assert(Src0);
291     int Src0Idx = NumOperands;
292     if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src0)) {
293       LLVM_DEBUG(dbgs() << "  failed: src0 is illegal\n");
294       Fail = true;
295       break;
296     }
297     DPPInst.add(*Src0);
298     DPPInst->getOperand(NumOperands).setIsKill(false);
299     ++NumOperands;
300 
301     if (auto *Mod1 = TII->getNamedOperand(OrigMI,
302                                           AMDGPU::OpName::src1_modifiers)) {
303       assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
304                                           AMDGPU::OpName::src1_modifiers));
305       assert(HasVOP3DPP ||
306              (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
307       DPPInst.addImm(Mod1->getImm());
308       ++NumOperands;
309     } else if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1_modifiers)) {
310       DPPInst.addImm(0);
311       ++NumOperands;
312     }
313     auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
314     if (Src1) {
315       int OpNum = NumOperands;
316       // If subtarget does not support SGPRs for src1 operand then the
317       // requirements are the same as for src0. We check src0 instead because
318       // pseudos are shared between subtargets and allow SGPR for src1 on all.
319       if (!ST->hasDPPSrc1SGPR()) {
320         assert(getOperandSize(*DPPInst, Src0Idx, *MRI) ==
321                    getOperandSize(*DPPInst, NumOperands, *MRI) &&
322                "Src0 and Src1 operands should have the same size");
323         OpNum = Src0Idx;
324       }
325       if (!TII->isOperandLegal(*DPPInst.getInstr(), OpNum, Src1)) {
326         LLVM_DEBUG(dbgs() << "  failed: src1 is illegal\n");
327         Fail = true;
328         break;
329       }
330       DPPInst.add(*Src1);
331       ++NumOperands;
332     }
333     if (auto *Mod2 =
334             TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2_modifiers)) {
335       assert(NumOperands ==
336              AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers));
337       assert(HasVOP3DPP ||
338              (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
339       DPPInst.addImm(Mod2->getImm());
340       ++NumOperands;
341     }
342     auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2);
343     if (Src2) {
344       if (!TII->getNamedOperand(*DPPInst.getInstr(), AMDGPU::OpName::src2) ||
345           !TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) {
346         LLVM_DEBUG(dbgs() << "  failed: src2 is illegal\n");
347         Fail = true;
348         break;
349       }
350       DPPInst.add(*Src2);
351       ++NumOperands;
352     }
353     if (HasVOP3DPP) {
354       auto *ClampOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::clamp);
355       if (ClampOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::clamp)) {
356         DPPInst.addImm(ClampOpr->getImm());
357       }
358       auto *VdstInOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst_in);
359       if (VdstInOpr &&
360           AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::vdst_in)) {
361         DPPInst.add(*VdstInOpr);
362       }
363       auto *OmodOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::omod);
364       if (OmodOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::omod)) {
365         DPPInst.addImm(OmodOpr->getImm());
366       }
367       // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to
368       // all 1.
369       if (auto *OpSelOpr =
370               TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel)) {
371         auto OpSel = OpSelOpr->getImm();
372         if (OpSel != 0) {
373           LLVM_DEBUG(dbgs() << "  failed: op_sel must be zero\n");
374           Fail = true;
375           break;
376         }
377         if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::op_sel))
378           DPPInst.addImm(OpSel);
379       }
380       if (auto *OpSelHiOpr =
381               TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel_hi)) {
382         auto OpSelHi = OpSelHiOpr->getImm();
383         // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check
384         // the bitmask for 3 op_sel_hi bits set
385         assert(Src2 && "Expected vop3p with 3 operands");
386         if (OpSelHi != 7) {
387           LLVM_DEBUG(dbgs() << "  failed: op_sel_hi must be all set to one\n");
388           Fail = true;
389           break;
390         }
391         if (AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::op_sel_hi))
392           DPPInst.addImm(OpSelHi);
393       }
394       auto *NegOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_lo);
395       if (NegOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::neg_lo)) {
396         DPPInst.addImm(NegOpr->getImm());
397       }
398       auto *NegHiOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_hi);
399       if (NegHiOpr && AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::neg_hi)) {
400         DPPInst.addImm(NegHiOpr->getImm());
401       }
402     }
403     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));
404     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));
405     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask));
406     DPPInst.addImm(CombBCZ ? 1 : 0);
407   } while (false);
408 
409   if (Fail) {
410     DPPInst.getInstr()->eraseFromParent();
411     return nullptr;
412   }
413   LLVM_DEBUG(dbgs() << "  combined:  " << *DPPInst.getInstr());
414   return DPPInst.getInstr();
415 }
416 
417 static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
418   assert(OldOpnd->isImm());
419   switch (OrigMIOp) {
420   default: break;
421   case AMDGPU::V_ADD_U32_e32:
422   case AMDGPU::V_ADD_U32_e64:
423   case AMDGPU::V_ADD_CO_U32_e32:
424   case AMDGPU::V_ADD_CO_U32_e64:
425   case AMDGPU::V_OR_B32_e32:
426   case AMDGPU::V_OR_B32_e64:
427   case AMDGPU::V_SUBREV_U32_e32:
428   case AMDGPU::V_SUBREV_U32_e64:
429   case AMDGPU::V_SUBREV_CO_U32_e32:
430   case AMDGPU::V_SUBREV_CO_U32_e64:
431   case AMDGPU::V_MAX_U32_e32:
432   case AMDGPU::V_MAX_U32_e64:
433   case AMDGPU::V_XOR_B32_e32:
434   case AMDGPU::V_XOR_B32_e64:
435     if (OldOpnd->getImm() == 0)
436       return true;
437     break;
438   case AMDGPU::V_AND_B32_e32:
439   case AMDGPU::V_AND_B32_e64:
440   case AMDGPU::V_MIN_U32_e32:
441   case AMDGPU::V_MIN_U32_e64:
442     if (static_cast<uint32_t>(OldOpnd->getImm()) ==
443         std::numeric_limits<uint32_t>::max())
444       return true;
445     break;
446   case AMDGPU::V_MIN_I32_e32:
447   case AMDGPU::V_MIN_I32_e64:
448     if (static_cast<int32_t>(OldOpnd->getImm()) ==
449         std::numeric_limits<int32_t>::max())
450       return true;
451     break;
452   case AMDGPU::V_MAX_I32_e32:
453   case AMDGPU::V_MAX_I32_e64:
454     if (static_cast<int32_t>(OldOpnd->getImm()) ==
455         std::numeric_limits<int32_t>::min())
456       return true;
457     break;
458   case AMDGPU::V_MUL_I32_I24_e32:
459   case AMDGPU::V_MUL_I32_I24_e64:
460   case AMDGPU::V_MUL_U32_U24_e32:
461   case AMDGPU::V_MUL_U32_U24_e64:
462     if (OldOpnd->getImm() == 1)
463       return true;
464     break;
465   }
466   return false;
467 }
468 
469 MachineInstr *GCNDPPCombine::createDPPInst(
470     MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
471     MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
472   assert(CombOldVGPR.Reg);
473   if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
474     auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
475     if (!Src1 || !Src1->isReg()) {
476       LLVM_DEBUG(dbgs() << "  failed: no src1 or it isn't a register\n");
477       return nullptr;
478     }
479     if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) {
480       LLVM_DEBUG(dbgs() << "  failed: old immediate isn't an identity\n");
481       return nullptr;
482     }
483     CombOldVGPR = getRegSubRegPair(*Src1);
484     auto MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
485     const TargetRegisterClass *RC = MRI->getRegClass(MovDst->getReg());
486     if (!isOfRegClass(CombOldVGPR, *RC, *MRI)) {
487       LLVM_DEBUG(dbgs() << "  failed: src1 has wrong register class\n");
488       return nullptr;
489     }
490   }
491   return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
492 }
493 
494 // returns true if MI doesn't have OpndName immediate operand or the
495 // operand has Value
496 bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName,
497                                     int64_t Value, int64_t Mask) const {
498   auto *Imm = TII->getNamedOperand(MI, OpndName);
499   if (!Imm)
500     return true;
501 
502   assert(Imm->isImm());
503   return (Imm->getImm() & Mask) == Value;
504 }
505 
506 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
507   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
508          MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
509          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
510   LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
511 
512   auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
513   assert(DstOpnd && DstOpnd->isReg());
514   auto DPPMovReg = DstOpnd->getReg();
515   if (DPPMovReg.isPhysical()) {
516     LLVM_DEBUG(dbgs() << "  failed: dpp move writes physreg\n");
517     return false;
518   }
519   if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) {
520     LLVM_DEBUG(dbgs() << "  failed: EXEC mask should remain the same"
521                          " for all uses\n");
522     return false;
523   }
524 
525   if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
526       MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
527     auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);
528     assert(DppCtrl && DppCtrl->isImm());
529     if (!AMDGPU::isLegalDPALU_DPPControl(DppCtrl->getImm())) {
530       LLVM_DEBUG(dbgs() << "  failed: 64 bit dpp move uses unsupported"
531                            " control value\n");
532       // Let it split, then control may become legal.
533       return false;
534     }
535   }
536 
537   auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
538   assert(RowMaskOpnd && RowMaskOpnd->isImm());
539   auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
540   assert(BankMaskOpnd && BankMaskOpnd->isImm());
541   const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
542                             BankMaskOpnd->getImm() == 0xF;
543 
544   auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);
545   assert(BCZOpnd && BCZOpnd->isImm());
546   bool BoundCtrlZero = BCZOpnd->getImm();
547 
548   auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old);
549   auto *SrcOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0);
550   assert(OldOpnd && OldOpnd->isReg());
551   assert(SrcOpnd && SrcOpnd->isReg());
552   if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {
553     LLVM_DEBUG(dbgs() << "  failed: dpp move reads physreg\n");
554     return false;
555   }
556 
557   auto * const OldOpndValue = getOldOpndValue(*OldOpnd);
558   // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
559   // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
560   // but the third option is used to distinguish undef from non-immediate
561   // to reuse IMPLICIT_DEF instruction later
562   assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
563 
564   bool CombBCZ = false;
565 
566   if (MaskAllLanes && BoundCtrlZero) { // [1]
567     CombBCZ = true;
568   } else {
569     if (!OldOpndValue || !OldOpndValue->isImm()) {
570       LLVM_DEBUG(dbgs() << "  failed: the DPP mov isn't combinable\n");
571       return false;
572     }
573 
574     if (OldOpndValue->getImm() == 0) {
575       if (MaskAllLanes) {
576         assert(!BoundCtrlZero); // by check [1]
577         CombBCZ = true;
578       }
579     } else if (BoundCtrlZero) {
580       assert(!MaskAllLanes); // by check [1]
581       LLVM_DEBUG(dbgs() <<
582         "  failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
583       return false;
584     }
585   }
586 
587   LLVM_DEBUG(dbgs() << "  old=";
588     if (!OldOpndValue)
589       dbgs() << "undef";
590     else
591       dbgs() << *OldOpndValue;
592     dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
593 
594   SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
595   DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos;
596   auto CombOldVGPR = getRegSubRegPair(*OldOpnd);
597   // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
598   if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
599     const TargetRegisterClass *RC = MRI->getRegClass(DPPMovReg);
600     CombOldVGPR = RegSubRegPair(
601       MRI->createVirtualRegister(RC));
602     auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(),
603                              TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg);
604     DPPMIs.push_back(UndefInst.getInstr());
605   }
606 
607   OrigMIs.push_back(&MovMI);
608   bool Rollback = true;
609   SmallVector<MachineOperand*, 16> Uses;
610 
611   for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) {
612     Uses.push_back(&Use);
613   }
614 
615   while (!Uses.empty()) {
616     MachineOperand *Use = Uses.pop_back_val();
617     Rollback = true;
618 
619     auto &OrigMI = *Use->getParent();
620     LLVM_DEBUG(dbgs() << "  try: " << OrigMI);
621 
622     auto OrigOp = OrigMI.getOpcode();
623     assert((TII->get(OrigOp).getSize() != 4 || !AMDGPU::isTrue16Inst(OrigOp)) &&
624            "There should not be e32 True16 instructions pre-RA");
625     if (OrigOp == AMDGPU::REG_SEQUENCE) {
626       Register FwdReg = OrigMI.getOperand(0).getReg();
627       unsigned FwdSubReg = 0;
628 
629       if (execMayBeModifiedBeforeAnyUse(*MRI, FwdReg, OrigMI)) {
630         LLVM_DEBUG(dbgs() << "  failed: EXEC mask should remain the same"
631                              " for all uses\n");
632         break;
633       }
634 
635       unsigned OpNo, E = OrigMI.getNumOperands();
636       for (OpNo = 1; OpNo < E; OpNo += 2) {
637         if (OrigMI.getOperand(OpNo).getReg() == DPPMovReg) {
638           FwdSubReg = OrigMI.getOperand(OpNo + 1).getImm();
639           break;
640         }
641       }
642 
643       if (!FwdSubReg)
644         break;
645 
646       for (auto &Op : MRI->use_nodbg_operands(FwdReg)) {
647         if (Op.getSubReg() == FwdSubReg)
648           Uses.push_back(&Op);
649       }
650       RegSeqWithOpNos[&OrigMI].push_back(OpNo);
651       continue;
652     }
653 
654     bool IsShrinkable = isShrinkable(OrigMI);
655     if (!(IsShrinkable ||
656           ((TII->isVOP3P(OrigOp) || TII->isVOPC(OrigOp) ||
657             TII->isVOP3(OrigOp)) &&
658            ST->hasVOP3DPP()) ||
659           TII->isVOP1(OrigOp) || TII->isVOP2(OrigOp))) {
660       LLVM_DEBUG(dbgs() << "  failed: not VOP1/2/3/3P/C\n");
661       break;
662     }
663     if (OrigMI.modifiesRegister(AMDGPU::EXEC, ST->getRegisterInfo())) {
664       LLVM_DEBUG(dbgs() << "  failed: can't combine v_cmpx\n");
665       break;
666     }
667 
668     auto *Src0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0);
669     auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
670     if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
671       LLVM_DEBUG(dbgs() << "  failed: no suitable operands\n");
672       break;
673     }
674 
675     auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2);
676     assert(Src0 && "Src1 without Src0?");
677     if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(*Src0)) ||
678                          (Src2 && Src2->isIdenticalTo(*Src0)))) ||
679         (Use == Src1 && (Src1->isIdenticalTo(*Src0) ||
680                          (Src2 && Src2->isIdenticalTo(*Src1))))) {
681       LLVM_DEBUG(
682           dbgs()
683           << "  " << OrigMI
684           << "  failed: DPP register is used more than once per instruction\n");
685       break;
686     }
687 
688     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
689     if (Use == Src0) {
690       if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
691                                         OldOpndValue, CombBCZ, IsShrinkable)) {
692         DPPMIs.push_back(DPPInst);
693         Rollback = false;
694       }
695     } else {
696       assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
697       auto *BB = OrigMI.getParent();
698       auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI);
699       BB->insert(OrigMI, NewMI);
700       if (TII->commuteInstruction(*NewMI)) {
701         LLVM_DEBUG(dbgs() << "  commuted:  " << *NewMI);
702         if (auto *DPPInst =
703                 createDPPInst(*NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
704                               IsShrinkable)) {
705           DPPMIs.push_back(DPPInst);
706           Rollback = false;
707         }
708       } else
709         LLVM_DEBUG(dbgs() << "  failed: cannot be commuted\n");
710       NewMI->eraseFromParent();
711     }
712     if (Rollback)
713       break;
714     OrigMIs.push_back(&OrigMI);
715   }
716 
717   Rollback |= !Uses.empty();
718 
719   for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
720     MI->eraseFromParent();
721 
722   if (!Rollback) {
723     for (auto &S : RegSeqWithOpNos) {
724       if (MRI->use_nodbg_empty(S.first->getOperand(0).getReg())) {
725         S.first->eraseFromParent();
726         continue;
727       }
728       while (!S.second.empty())
729         S.first->getOperand(S.second.pop_back_val()).setIsUndef();
730     }
731   }
732 
733   return !Rollback;
734 }
735 
736 bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
737   ST = &MF.getSubtarget<GCNSubtarget>();
738   if (!ST->hasDPP() || skipFunction(MF.getFunction()))
739     return false;
740 
741   MRI = &MF.getRegInfo();
742   TII = ST->getInstrInfo();
743 
744   bool Changed = false;
745   for (auto &MBB : MF) {
746     for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) {
747       if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) {
748         Changed = true;
749         ++NumDPPMovsCombined;
750       } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
751                  MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
752         if (ST->hasDPALU_DPP() && combineDPPMov(MI)) {
753           Changed = true;
754           ++NumDPPMovsCombined;
755         } else {
756           auto Split = TII->expandMovDPP64(MI);
757           for (auto *M : {Split.first, Split.second}) {
758             if (M && combineDPPMov(*M))
759               ++NumDPPMovsCombined;
760           }
761           Changed = true;
762         }
763       }
764     }
765   }
766   return Changed;
767 }
768