1 //===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 /// \file
8 //===----------------------------------------------------------------------===//
9 //
10
11 #include "AMDGPU.h"
12 #include "GCNSubtarget.h"
13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14 #include "SIMachineFunctionInfo.h"
15 #include "llvm/ADT/DepthFirstIterator.h"
16 #include "llvm/CodeGen/MachineFunctionPass.h"
17 #include "llvm/CodeGen/MachineOperand.h"
18
19 #define DEBUG_TYPE "si-fold-operands"
20 using namespace llvm;
21
22 namespace {
23
24 struct FoldCandidate {
25 MachineInstr *UseMI;
26 union {
27 MachineOperand *OpToFold;
28 uint64_t ImmToFold;
29 int FrameIndexToFold;
30 };
31 int ShrinkOpcode;
32 unsigned UseOpNo;
33 MachineOperand::MachineOperandType Kind;
34 bool Commuted;
35
FoldCandidate__anon66c9ee070111::FoldCandidate36 FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp,
37 bool Commuted_ = false,
38 int ShrinkOp = -1) :
39 UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
40 Kind(FoldOp->getType()),
41 Commuted(Commuted_) {
42 if (FoldOp->isImm()) {
43 ImmToFold = FoldOp->getImm();
44 } else if (FoldOp->isFI()) {
45 FrameIndexToFold = FoldOp->getIndex();
46 } else {
47 assert(FoldOp->isReg() || FoldOp->isGlobal());
48 OpToFold = FoldOp;
49 }
50 }
51
isFI__anon66c9ee070111::FoldCandidate52 bool isFI() const {
53 return Kind == MachineOperand::MO_FrameIndex;
54 }
55
isImm__anon66c9ee070111::FoldCandidate56 bool isImm() const {
57 return Kind == MachineOperand::MO_Immediate;
58 }
59
isReg__anon66c9ee070111::FoldCandidate60 bool isReg() const {
61 return Kind == MachineOperand::MO_Register;
62 }
63
isGlobal__anon66c9ee070111::FoldCandidate64 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
65
needsShrink__anon66c9ee070111::FoldCandidate66 bool needsShrink() const { return ShrinkOpcode != -1; }
67 };
68
69 class SIFoldOperands : public MachineFunctionPass {
70 public:
71 static char ID;
72 MachineRegisterInfo *MRI;
73 const SIInstrInfo *TII;
74 const SIRegisterInfo *TRI;
75 const GCNSubtarget *ST;
76 const SIMachineFunctionInfo *MFI;
77
78 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
79 const MachineOperand &OpToFold) const;
80
81 bool updateOperand(FoldCandidate &Fold) const;
82
83 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
84 MachineInstr *MI, unsigned OpNo,
85 MachineOperand *OpToFold) const;
86 bool isUseSafeToFold(const MachineInstr &MI,
87 const MachineOperand &UseMO) const;
88 bool
89 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
90 Register UseReg, uint8_t OpTy) const;
91 bool tryToFoldACImm(const MachineOperand &OpToFold, MachineInstr *UseMI,
92 unsigned UseOpIdx,
93 SmallVectorImpl<FoldCandidate> &FoldList) const;
94 void foldOperand(MachineOperand &OpToFold,
95 MachineInstr *UseMI,
96 int UseOpIdx,
97 SmallVectorImpl<FoldCandidate> &FoldList,
98 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
99
100 MachineOperand *getImmOrMaterializedImm(MachineOperand &Op) const;
101 bool tryConstantFoldOp(MachineInstr *MI) const;
102 bool tryFoldCndMask(MachineInstr &MI) const;
103 bool tryFoldZeroHighBits(MachineInstr &MI) const;
104 bool foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;
105 bool tryFoldFoldableCopy(MachineInstr &MI,
106 MachineOperand *&CurrentKnownM0Val) const;
107
108 const MachineOperand *isClamp(const MachineInstr &MI) const;
109 bool tryFoldClamp(MachineInstr &MI);
110
111 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
112 bool tryFoldOMod(MachineInstr &MI);
113 bool tryFoldRegSequence(MachineInstr &MI);
114 bool tryFoldLCSSAPhi(MachineInstr &MI);
115 bool tryFoldLoad(MachineInstr &MI);
116
117 public:
SIFoldOperands()118 SIFoldOperands() : MachineFunctionPass(ID) {
119 initializeSIFoldOperandsPass(*PassRegistry::getPassRegistry());
120 }
121
122 bool runOnMachineFunction(MachineFunction &MF) override;
123
getPassName() const124 StringRef getPassName() const override { return "SI Fold Operands"; }
125
getAnalysisUsage(AnalysisUsage & AU) const126 void getAnalysisUsage(AnalysisUsage &AU) const override {
127 AU.setPreservesCFG();
128 MachineFunctionPass::getAnalysisUsage(AU);
129 }
130 };
131
132 } // End anonymous namespace.
133
134 INITIALIZE_PASS(SIFoldOperands, DEBUG_TYPE,
135 "SI Fold Operands", false, false)
136
137 char SIFoldOperands::ID = 0;
138
139 char &llvm::SIFoldOperandsID = SIFoldOperands::ID;
140
141 // Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
macToMad(unsigned Opc)142 static unsigned macToMad(unsigned Opc) {
143 switch (Opc) {
144 case AMDGPU::V_MAC_F32_e64:
145 return AMDGPU::V_MAD_F32_e64;
146 case AMDGPU::V_MAC_F16_e64:
147 return AMDGPU::V_MAD_F16_e64;
148 case AMDGPU::V_FMAC_F32_e64:
149 return AMDGPU::V_FMA_F32_e64;
150 case AMDGPU::V_FMAC_F16_e64:
151 return AMDGPU::V_FMA_F16_gfx9_e64;
152 case AMDGPU::V_FMAC_F16_t16_e64:
153 return AMDGPU::V_FMA_F16_gfx9_e64;
154 case AMDGPU::V_FMAC_LEGACY_F32_e64:
155 return AMDGPU::V_FMA_LEGACY_F32_e64;
156 case AMDGPU::V_FMAC_F64_e64:
157 return AMDGPU::V_FMA_F64_e64;
158 }
159 return AMDGPU::INSTRUCTION_LIST_END;
160 }
161
162 // TODO: Add heuristic that the frame index might not fit in the addressing mode
163 // immediate offset to avoid materializing in loops.
frameIndexMayFold(const MachineInstr & UseMI,int OpNo,const MachineOperand & OpToFold) const164 bool SIFoldOperands::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
165 const MachineOperand &OpToFold) const {
166 if (!OpToFold.isFI())
167 return false;
168
169 const unsigned Opc = UseMI.getOpcode();
170 if (TII->isMUBUF(UseMI))
171 return OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
172 if (!TII->isFLATScratch(UseMI))
173 return false;
174
175 int SIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
176 if (OpNo == SIdx)
177 return true;
178
179 int VIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
180 return OpNo == VIdx && SIdx == -1;
181 }
182
createSIFoldOperandsPass()183 FunctionPass *llvm::createSIFoldOperandsPass() {
184 return new SIFoldOperands();
185 }
186
updateOperand(FoldCandidate & Fold) const187 bool SIFoldOperands::updateOperand(FoldCandidate &Fold) const {
188 MachineInstr *MI = Fold.UseMI;
189 MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
190 assert(Old.isReg());
191
192
193 const uint64_t TSFlags = MI->getDesc().TSFlags;
194 if (Fold.isImm()) {
195 if (TSFlags & SIInstrFlags::IsPacked && !(TSFlags & SIInstrFlags::IsMAI) &&
196 (!ST->hasDOTOpSelHazard() || !(TSFlags & SIInstrFlags::IsDOT)) &&
197 AMDGPU::isFoldableLiteralV216(Fold.ImmToFold,
198 ST->hasInv2PiInlineImm())) {
199 // Set op_sel/op_sel_hi on this operand or bail out if op_sel is
200 // already set.
201 unsigned Opcode = MI->getOpcode();
202 int OpNo = MI->getOperandNo(&Old);
203 int ModIdx = -1;
204 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0))
205 ModIdx = AMDGPU::OpName::src0_modifiers;
206 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1))
207 ModIdx = AMDGPU::OpName::src1_modifiers;
208 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2))
209 ModIdx = AMDGPU::OpName::src2_modifiers;
210 assert(ModIdx != -1);
211 ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx);
212 MachineOperand &Mod = MI->getOperand(ModIdx);
213 unsigned Val = Mod.getImm();
214 if (!(Val & SISrcMods::OP_SEL_0) && (Val & SISrcMods::OP_SEL_1)) {
215 // Only apply the following transformation if that operand requires
216 // a packed immediate.
217 switch (TII->get(Opcode).operands()[OpNo].OperandType) {
218 case AMDGPU::OPERAND_REG_IMM_V2FP16:
219 case AMDGPU::OPERAND_REG_IMM_V2INT16:
220 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
221 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
222 // If upper part is all zero we do not need op_sel_hi.
223 if (!isUInt<16>(Fold.ImmToFold)) {
224 if (!(Fold.ImmToFold & 0xffff)) {
225 Mod.setImm(Mod.getImm() | SISrcMods::OP_SEL_0);
226 Mod.setImm(Mod.getImm() & ~SISrcMods::OP_SEL_1);
227 Old.ChangeToImmediate((Fold.ImmToFold >> 16) & 0xffff);
228 return true;
229 }
230 Mod.setImm(Mod.getImm() & ~SISrcMods::OP_SEL_1);
231 Old.ChangeToImmediate(Fold.ImmToFold & 0xffff);
232 return true;
233 }
234 break;
235 default:
236 break;
237 }
238 }
239 }
240 }
241
242 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
243 MachineBasicBlock *MBB = MI->getParent();
244 auto Liveness = MBB->computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 16);
245 if (Liveness != MachineBasicBlock::LQR_Dead) {
246 LLVM_DEBUG(dbgs() << "Not shrinking " << MI << " due to vcc liveness\n");
247 return false;
248 }
249
250 int Op32 = Fold.ShrinkOpcode;
251 MachineOperand &Dst0 = MI->getOperand(0);
252 MachineOperand &Dst1 = MI->getOperand(1);
253 assert(Dst0.isDef() && Dst1.isDef());
254
255 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(Dst1.getReg());
256
257 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Dst0.getReg());
258 Register NewReg0 = MRI->createVirtualRegister(Dst0RC);
259
260 MachineInstr *Inst32 = TII->buildShrunkInst(*MI, Op32);
261
262 if (HaveNonDbgCarryUse) {
263 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::COPY),
264 Dst1.getReg())
265 .addReg(AMDGPU::VCC, RegState::Kill);
266 }
267
268 // Keep the old instruction around to avoid breaking iterators, but
269 // replace it with a dummy instruction to remove uses.
270 //
271 // FIXME: We should not invert how this pass looks at operands to avoid
272 // this. Should track set of foldable movs instead of looking for uses
273 // when looking at a use.
274 Dst0.setReg(NewReg0);
275 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
276 MI->removeOperand(I);
277 MI->setDesc(TII->get(AMDGPU::IMPLICIT_DEF));
278
279 if (Fold.Commuted)
280 TII->commuteInstruction(*Inst32, false);
281 return true;
282 }
283
284 assert(!Fold.needsShrink() && "not handled");
285
286 if (Fold.isImm()) {
287 if (Old.isTied()) {
288 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(MI->getOpcode());
289 if (NewMFMAOpc == -1)
290 return false;
291 MI->setDesc(TII->get(NewMFMAOpc));
292 MI->untieRegOperand(0);
293 }
294 Old.ChangeToImmediate(Fold.ImmToFold);
295 return true;
296 }
297
298 if (Fold.isGlobal()) {
299 Old.ChangeToGA(Fold.OpToFold->getGlobal(), Fold.OpToFold->getOffset(),
300 Fold.OpToFold->getTargetFlags());
301 return true;
302 }
303
304 if (Fold.isFI()) {
305 Old.ChangeToFrameIndex(Fold.FrameIndexToFold);
306 return true;
307 }
308
309 MachineOperand *New = Fold.OpToFold;
310 Old.substVirtReg(New->getReg(), New->getSubReg(), *TRI);
311 Old.setIsUndef(New->isUndef());
312 return true;
313 }
314
isUseMIInFoldList(ArrayRef<FoldCandidate> FoldList,const MachineInstr * MI)315 static bool isUseMIInFoldList(ArrayRef<FoldCandidate> FoldList,
316 const MachineInstr *MI) {
317 return any_of(FoldList, [&](const auto &C) { return C.UseMI == MI; });
318 }
319
appendFoldCandidate(SmallVectorImpl<FoldCandidate> & FoldList,MachineInstr * MI,unsigned OpNo,MachineOperand * FoldOp,bool Commuted=false,int ShrinkOp=-1)320 static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
321 MachineInstr *MI, unsigned OpNo,
322 MachineOperand *FoldOp, bool Commuted = false,
323 int ShrinkOp = -1) {
324 // Skip additional folding on the same operand.
325 for (FoldCandidate &Fold : FoldList)
326 if (Fold.UseMI == MI && Fold.UseOpNo == OpNo)
327 return;
328 LLVM_DEBUG(dbgs() << "Append " << (Commuted ? "commuted" : "normal")
329 << " operand " << OpNo << "\n " << *MI);
330 FoldList.emplace_back(MI, OpNo, FoldOp, Commuted, ShrinkOp);
331 }
332
tryAddToFoldList(SmallVectorImpl<FoldCandidate> & FoldList,MachineInstr * MI,unsigned OpNo,MachineOperand * OpToFold) const333 bool SIFoldOperands::tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
334 MachineInstr *MI, unsigned OpNo,
335 MachineOperand *OpToFold) const {
336 if (!TII->isOperandLegal(*MI, OpNo, OpToFold)) {
337 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
338 unsigned Opc = MI->getOpcode();
339 unsigned NewOpc = macToMad(Opc);
340 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
341 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
342 // to fold the operand.
343 MI->setDesc(TII->get(NewOpc));
344 if (!AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::op_sel) &&
345 AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel))
346 MI->addOperand(MachineOperand::CreateImm(0));
347 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
348 if (FoldAsMAD) {
349 MI->untieRegOperand(OpNo);
350 return true;
351 }
352 MI->setDesc(TII->get(Opc));
353 }
354
355 // Special case for s_setreg_b32
356 if (OpToFold->isImm()) {
357 unsigned ImmOpc = 0;
358 if (Opc == AMDGPU::S_SETREG_B32)
359 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
360 else if (Opc == AMDGPU::S_SETREG_B32_mode)
361 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
362 if (ImmOpc) {
363 MI->setDesc(TII->get(ImmOpc));
364 appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
365 return true;
366 }
367 }
368
369 // If we are already folding into another operand of MI, then
370 // we can't commute the instruction, otherwise we risk making the
371 // other fold illegal.
372 if (isUseMIInFoldList(FoldList, MI))
373 return false;
374
375 unsigned CommuteOpNo = OpNo;
376
377 // Operand is not legal, so try to commute the instruction to
378 // see if this makes it possible to fold.
379 unsigned CommuteIdx0 = TargetInstrInfo::CommuteAnyOperandIndex;
380 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
381 bool CanCommute = TII->findCommutedOpIndices(*MI, CommuteIdx0, CommuteIdx1);
382
383 if (CanCommute) {
384 if (CommuteIdx0 == OpNo)
385 CommuteOpNo = CommuteIdx1;
386 else if (CommuteIdx1 == OpNo)
387 CommuteOpNo = CommuteIdx0;
388 }
389
390
391 // One of operands might be an Imm operand, and OpNo may refer to it after
392 // the call of commuteInstruction() below. Such situations are avoided
393 // here explicitly as OpNo must be a register operand to be a candidate
394 // for memory folding.
395 if (CanCommute && (!MI->getOperand(CommuteIdx0).isReg() ||
396 !MI->getOperand(CommuteIdx1).isReg()))
397 return false;
398
399 if (!CanCommute ||
400 !TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1))
401 return false;
402
403 if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) {
404 if ((Opc == AMDGPU::V_ADD_CO_U32_e64 ||
405 Opc == AMDGPU::V_SUB_CO_U32_e64 ||
406 Opc == AMDGPU::V_SUBREV_CO_U32_e64) && // FIXME
407 (OpToFold->isImm() || OpToFold->isFI() || OpToFold->isGlobal())) {
408
409 // Verify the other operand is a VGPR, otherwise we would violate the
410 // constant bus restriction.
411 unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0;
412 MachineOperand &OtherOp = MI->getOperand(OtherIdx);
413 if (!OtherOp.isReg() ||
414 !TII->getRegisterInfo().isVGPR(*MRI, OtherOp.getReg()))
415 return false;
416
417 assert(MI->getOperand(1).isDef());
418
419 // Make sure to get the 32-bit version of the commuted opcode.
420 unsigned MaybeCommutedOpc = MI->getOpcode();
421 int Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc);
422
423 appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true, Op32);
424 return true;
425 }
426
427 TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1);
428 return false;
429 }
430
431 appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true);
432 return true;
433 }
434
435 // Check the case where we might introduce a second constant operand to a
436 // scalar instruction
437 if (TII->isSALU(MI->getOpcode())) {
438 const MCInstrDesc &InstDesc = MI->getDesc();
439 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
440
441 // Fine if the operand can be encoded as an inline constant
442 if (!OpToFold->isReg() && !TII->isInlineConstant(*OpToFold, OpInfo)) {
443 // Otherwise check for another constant
444 for (unsigned i = 0, e = InstDesc.getNumOperands(); i != e; ++i) {
445 auto &Op = MI->getOperand(i);
446 if (OpNo != i && !Op.isReg() && !TII->isInlineConstant(Op, OpInfo))
447 return false;
448 }
449 }
450 }
451
452 appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
453 return true;
454 }
455
isUseSafeToFold(const MachineInstr & MI,const MachineOperand & UseMO) const456 bool SIFoldOperands::isUseSafeToFold(const MachineInstr &MI,
457 const MachineOperand &UseMO) const {
458 // Operands of SDWA instructions must be registers.
459 return !TII->isSDWA(MI);
460 }
461
462 // Find a def of the UseReg, check if it is a reg_sequence and find initializers
463 // for each subreg, tracking it to foldable inline immediate if possible.
464 // Returns true on success.
getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *,unsigned>> & Defs,Register UseReg,uint8_t OpTy) const465 bool SIFoldOperands::getRegSeqInit(
466 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
467 Register UseReg, uint8_t OpTy) const {
468 MachineInstr *Def = MRI->getVRegDef(UseReg);
469 if (!Def || !Def->isRegSequence())
470 return false;
471
472 for (unsigned I = 1, E = Def->getNumExplicitOperands(); I < E; I += 2) {
473 MachineOperand *Sub = &Def->getOperand(I);
474 assert(Sub->isReg());
475
476 for (MachineInstr *SubDef = MRI->getVRegDef(Sub->getReg());
477 SubDef && Sub->isReg() && Sub->getReg().isVirtual() &&
478 !Sub->getSubReg() && TII->isFoldableCopy(*SubDef);
479 SubDef = MRI->getVRegDef(Sub->getReg())) {
480 MachineOperand *Op = &SubDef->getOperand(1);
481 if (Op->isImm()) {
482 if (TII->isInlineConstant(*Op, OpTy))
483 Sub = Op;
484 break;
485 }
486 if (!Op->isReg() || Op->getReg().isPhysical())
487 break;
488 Sub = Op;
489 }
490
491 Defs.emplace_back(Sub, Def->getOperand(I + 1).getImm());
492 }
493
494 return true;
495 }
496
tryToFoldACImm(const MachineOperand & OpToFold,MachineInstr * UseMI,unsigned UseOpIdx,SmallVectorImpl<FoldCandidate> & FoldList) const497 bool SIFoldOperands::tryToFoldACImm(
498 const MachineOperand &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
499 SmallVectorImpl<FoldCandidate> &FoldList) const {
500 const MCInstrDesc &Desc = UseMI->getDesc();
501 if (UseOpIdx >= Desc.getNumOperands())
502 return false;
503
504 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
505 if ((OpTy < AMDGPU::OPERAND_REG_INLINE_AC_FIRST ||
506 OpTy > AMDGPU::OPERAND_REG_INLINE_AC_LAST) &&
507 (OpTy < AMDGPU::OPERAND_REG_INLINE_C_FIRST ||
508 OpTy > AMDGPU::OPERAND_REG_INLINE_C_LAST))
509 return false;
510
511 if (OpToFold.isImm() && TII->isInlineConstant(OpToFold, OpTy) &&
512 TII->isOperandLegal(*UseMI, UseOpIdx, &OpToFold)) {
513 UseMI->getOperand(UseOpIdx).ChangeToImmediate(OpToFold.getImm());
514 return true;
515 }
516
517 if (!OpToFold.isReg())
518 return false;
519
520 Register UseReg = OpToFold.getReg();
521 if (!UseReg.isVirtual())
522 return false;
523
524 if (isUseMIInFoldList(FoldList, UseMI))
525 return false;
526
527 // Maybe it is just a COPY of an immediate itself.
528 MachineInstr *Def = MRI->getVRegDef(UseReg);
529 MachineOperand &UseOp = UseMI->getOperand(UseOpIdx);
530 if (!UseOp.getSubReg() && Def && TII->isFoldableCopy(*Def)) {
531 MachineOperand &DefOp = Def->getOperand(1);
532 if (DefOp.isImm() && TII->isInlineConstant(DefOp, OpTy) &&
533 TII->isOperandLegal(*UseMI, UseOpIdx, &DefOp)) {
534 UseMI->getOperand(UseOpIdx).ChangeToImmediate(DefOp.getImm());
535 return true;
536 }
537 }
538
539 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
540 if (!getRegSeqInit(Defs, UseReg, OpTy))
541 return false;
542
543 int32_t Imm;
544 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
545 const MachineOperand *Op = Defs[I].first;
546 if (!Op->isImm())
547 return false;
548
549 auto SubImm = Op->getImm();
550 if (!I) {
551 Imm = SubImm;
552 if (!TII->isInlineConstant(*Op, OpTy) ||
553 !TII->isOperandLegal(*UseMI, UseOpIdx, Op))
554 return false;
555
556 continue;
557 }
558 if (Imm != SubImm)
559 return false; // Can only fold splat constants
560 }
561
562 appendFoldCandidate(FoldList, UseMI, UseOpIdx, Defs[0].first);
563 return true;
564 }
565
foldOperand(MachineOperand & OpToFold,MachineInstr * UseMI,int UseOpIdx,SmallVectorImpl<FoldCandidate> & FoldList,SmallVectorImpl<MachineInstr * > & CopiesToReplace) const566 void SIFoldOperands::foldOperand(
567 MachineOperand &OpToFold,
568 MachineInstr *UseMI,
569 int UseOpIdx,
570 SmallVectorImpl<FoldCandidate> &FoldList,
571 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
572 const MachineOperand &UseOp = UseMI->getOperand(UseOpIdx);
573
574 if (!isUseSafeToFold(*UseMI, UseOp))
575 return;
576
577 // FIXME: Fold operands with subregs.
578 if (UseOp.isReg() && OpToFold.isReg() &&
579 (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister))
580 return;
581
582 // Special case for REG_SEQUENCE: We can't fold literals into
583 // REG_SEQUENCE instructions, so we have to fold them into the
584 // uses of REG_SEQUENCE.
585 if (UseMI->isRegSequence()) {
586 Register RegSeqDstReg = UseMI->getOperand(0).getReg();
587 unsigned RegSeqDstSubReg = UseMI->getOperand(UseOpIdx + 1).getImm();
588
589 for (auto &RSUse : make_early_inc_range(MRI->use_nodbg_operands(RegSeqDstReg))) {
590 MachineInstr *RSUseMI = RSUse.getParent();
591
592 if (tryToFoldACImm(UseMI->getOperand(0), RSUseMI,
593 RSUseMI->getOperandNo(&RSUse), FoldList))
594 continue;
595
596 if (RSUse.getSubReg() != RegSeqDstSubReg)
597 continue;
598
599 foldOperand(OpToFold, RSUseMI, RSUseMI->getOperandNo(&RSUse), FoldList,
600 CopiesToReplace);
601 }
602
603 return;
604 }
605
606 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
607 return;
608
609 if (frameIndexMayFold(*UseMI, UseOpIdx, OpToFold)) {
610 // Verify that this is a stack access.
611 // FIXME: Should probably use stack pseudos before frame lowering.
612
613 if (TII->isMUBUF(*UseMI)) {
614 if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() !=
615 MFI->getScratchRSrcReg())
616 return;
617
618 // Ensure this is either relative to the current frame or the current
619 // wave.
620 MachineOperand &SOff =
621 *TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset);
622 if (!SOff.isImm() || SOff.getImm() != 0)
623 return;
624 }
625
626 // A frame index will resolve to a positive constant, so it should always be
627 // safe to fold the addressing mode, even pre-GFX9.
628 UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex());
629
630 const unsigned Opc = UseMI->getOpcode();
631 if (TII->isFLATScratch(*UseMI) &&
632 AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr) &&
633 !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::saddr)) {
634 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opc);
635 UseMI->setDesc(TII->get(NewOpc));
636 }
637
638 return;
639 }
640
641 bool FoldingImmLike =
642 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
643
644 if (FoldingImmLike && UseMI->isCopy()) {
645 Register DestReg = UseMI->getOperand(0).getReg();
646 Register SrcReg = UseMI->getOperand(1).getReg();
647 assert(SrcReg.isVirtual());
648
649 const TargetRegisterClass *SrcRC = MRI->getRegClass(SrcReg);
650
651 // Don't fold into a copy to a physical register with the same class. Doing
652 // so would interfere with the register coalescer's logic which would avoid
653 // redundant initializations.
654 if (DestReg.isPhysical() && SrcRC->contains(DestReg))
655 return;
656
657 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(*MRI, DestReg);
658 if (!DestReg.isPhysical()) {
659 if (TRI->isSGPRClass(SrcRC) && TRI->hasVectorRegisters(DestRC)) {
660 SmallVector<FoldCandidate, 4> CopyUses;
661 for (auto &Use : MRI->use_nodbg_operands(DestReg)) {
662 // There's no point trying to fold into an implicit operand.
663 if (Use.isImplicit())
664 continue;
665
666 CopyUses.emplace_back(Use.getParent(),
667 Use.getParent()->getOperandNo(&Use),
668 &UseMI->getOperand(1));
669 }
670
671 for (auto &F : CopyUses) {
672 foldOperand(*F.OpToFold, F.UseMI, F.UseOpNo, FoldList,
673 CopiesToReplace);
674 }
675 }
676
677 if (DestRC == &AMDGPU::AGPR_32RegClass &&
678 TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
679 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64));
680 UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
681 CopiesToReplace.push_back(UseMI);
682 return;
683 }
684 }
685
686 // In order to fold immediates into copies, we need to change the
687 // copy to a MOV.
688
689 unsigned MovOp = TII->getMovOpcode(DestRC);
690 if (MovOp == AMDGPU::COPY)
691 return;
692
693 UseMI->setDesc(TII->get(MovOp));
694 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
695 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
696 while (ImpOpI != ImpOpE) {
697 MachineInstr::mop_iterator Tmp = ImpOpI;
698 ImpOpI++;
699 UseMI->removeOperand(UseMI->getOperandNo(Tmp));
700 }
701 CopiesToReplace.push_back(UseMI);
702 } else {
703 if (UseMI->isCopy() && OpToFold.isReg() &&
704 UseMI->getOperand(0).getReg().isVirtual() &&
705 !UseMI->getOperand(1).getSubReg()) {
706 LLVM_DEBUG(dbgs() << "Folding " << OpToFold << "\n into " << *UseMI);
707 unsigned Size = TII->getOpSize(*UseMI, 1);
708 Register UseReg = OpToFold.getReg();
709 UseMI->getOperand(1).setReg(UseReg);
710 UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
711 UseMI->getOperand(1).setIsKill(false);
712 CopiesToReplace.push_back(UseMI);
713 OpToFold.setIsKill(false);
714
715 // Remove kill flags as kills may now be out of order with uses.
716 MRI->clearKillFlags(OpToFold.getReg());
717
718 // That is very tricky to store a value into an AGPR. v_accvgpr_write_b32
719 // can only accept VGPR or inline immediate. Recreate a reg_sequence with
720 // its initializers right here, so we will rematerialize immediates and
721 // avoid copies via different reg classes.
722 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
723 if (Size > 4 && TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) &&
724 getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
725 const DebugLoc &DL = UseMI->getDebugLoc();
726 MachineBasicBlock &MBB = *UseMI->getParent();
727
728 UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE));
729 for (unsigned I = UseMI->getNumOperands() - 1; I > 0; --I)
730 UseMI->removeOperand(I);
731
732 MachineInstrBuilder B(*MBB.getParent(), UseMI);
733 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
734 SmallSetVector<TargetInstrInfo::RegSubRegPair, 32> SeenAGPRs;
735 for (unsigned I = 0; I < Size / 4; ++I) {
736 MachineOperand *Def = Defs[I].first;
737 TargetInstrInfo::RegSubRegPair CopyToVGPR;
738 if (Def->isImm() &&
739 TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
740 int64_t Imm = Def->getImm();
741
742 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
743 BuildMI(MBB, UseMI, DL,
744 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm);
745 B.addReg(Tmp);
746 } else if (Def->isReg() && TRI->isAGPR(*MRI, Def->getReg())) {
747 auto Src = getRegSubRegPair(*Def);
748 Def->setIsKill(false);
749 if (!SeenAGPRs.insert(Src)) {
750 // We cannot build a reg_sequence out of the same registers, they
751 // must be copied. Better do it here before copyPhysReg() created
752 // several reads to do the AGPR->VGPR->AGPR copy.
753 CopyToVGPR = Src;
754 } else {
755 B.addReg(Src.Reg, Def->isUndef() ? RegState::Undef : 0,
756 Src.SubReg);
757 }
758 } else {
759 assert(Def->isReg());
760 Def->setIsKill(false);
761 auto Src = getRegSubRegPair(*Def);
762
763 // Direct copy from SGPR to AGPR is not possible. To avoid creation
764 // of exploded copies SGPR->VGPR->AGPR in the copyPhysReg() later,
765 // create a copy here and track if we already have such a copy.
766 if (TRI->isSGPRReg(*MRI, Src.Reg)) {
767 CopyToVGPR = Src;
768 } else {
769 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
770 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def);
771 B.addReg(Tmp);
772 }
773 }
774
775 if (CopyToVGPR.Reg) {
776 Register Vgpr;
777 if (VGPRCopies.count(CopyToVGPR)) {
778 Vgpr = VGPRCopies[CopyToVGPR];
779 } else {
780 Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
781 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def);
782 VGPRCopies[CopyToVGPR] = Vgpr;
783 }
784 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass);
785 BuildMI(MBB, UseMI, DL,
786 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addReg(Vgpr);
787 B.addReg(Tmp);
788 }
789
790 B.addImm(Defs[I].second);
791 }
792 LLVM_DEBUG(dbgs() << "Folded " << *UseMI);
793 return;
794 }
795
796 if (Size != 4)
797 return;
798
799 Register Reg0 = UseMI->getOperand(0).getReg();
800 Register Reg1 = UseMI->getOperand(1).getReg();
801 if (TRI->isAGPR(*MRI, Reg0) && TRI->isVGPR(*MRI, Reg1))
802 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64));
803 else if (TRI->isVGPR(*MRI, Reg0) && TRI->isAGPR(*MRI, Reg1))
804 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64));
805 else if (ST->hasGFX90AInsts() && TRI->isAGPR(*MRI, Reg0) &&
806 TRI->isAGPR(*MRI, Reg1))
807 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_MOV_B32));
808 return;
809 }
810
811 unsigned UseOpc = UseMI->getOpcode();
812 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
813 (UseOpc == AMDGPU::V_READLANE_B32 &&
814 (int)UseOpIdx ==
815 AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) {
816 // %vgpr = V_MOV_B32 imm
817 // %sgpr = V_READFIRSTLANE_B32 %vgpr
818 // =>
819 // %sgpr = S_MOV_B32 imm
820 if (FoldingImmLike) {
821 if (execMayBeModifiedBeforeUse(*MRI,
822 UseMI->getOperand(UseOpIdx).getReg(),
823 *OpToFold.getParent(),
824 *UseMI))
825 return;
826
827 UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32));
828
829 if (OpToFold.isImm())
830 UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
831 else
832 UseMI->getOperand(1).ChangeToFrameIndex(OpToFold.getIndex());
833 UseMI->removeOperand(2); // Remove exec read (or src1 for readlane)
834 return;
835 }
836
837 if (OpToFold.isReg() && TRI->isSGPRReg(*MRI, OpToFold.getReg())) {
838 if (execMayBeModifiedBeforeUse(*MRI,
839 UseMI->getOperand(UseOpIdx).getReg(),
840 *OpToFold.getParent(),
841 *UseMI))
842 return;
843
844 // %vgpr = COPY %sgpr0
845 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
846 // =>
847 // %sgpr1 = COPY %sgpr0
848 UseMI->setDesc(TII->get(AMDGPU::COPY));
849 UseMI->getOperand(1).setReg(OpToFold.getReg());
850 UseMI->getOperand(1).setSubReg(OpToFold.getSubReg());
851 UseMI->getOperand(1).setIsKill(false);
852 UseMI->removeOperand(2); // Remove exec read (or src1 for readlane)
853 return;
854 }
855 }
856
857 const MCInstrDesc &UseDesc = UseMI->getDesc();
858
859 // Don't fold into target independent nodes. Target independent opcodes
860 // don't have defined register classes.
861 if (UseDesc.isVariadic() || UseOp.isImplicit() ||
862 UseDesc.operands()[UseOpIdx].RegClass == -1)
863 return;
864 }
865
866 if (!FoldingImmLike) {
867 if (OpToFold.isReg() && ST->needsAlignedVGPRs()) {
868 // Don't fold if OpToFold doesn't hold an aligned register.
869 const TargetRegisterClass *RC =
870 TRI->getRegClassForReg(*MRI, OpToFold.getReg());
871 if (TRI->hasVectorRegisters(RC) && OpToFold.getSubReg()) {
872 unsigned SubReg = OpToFold.getSubReg();
873 if (const TargetRegisterClass *SubRC =
874 TRI->getSubRegisterClass(RC, SubReg))
875 RC = SubRC;
876 }
877
878 if (!RC || !TRI->isProperlyAlignedRC(*RC))
879 return;
880 }
881
882 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold);
883
884 // FIXME: We could try to change the instruction from 64-bit to 32-bit
885 // to enable more folding opportunities. The shrink operands pass
886 // already does this.
887 return;
888 }
889
890
891 const MCInstrDesc &FoldDesc = OpToFold.getParent()->getDesc();
892 const TargetRegisterClass *FoldRC =
893 TRI->getRegClass(FoldDesc.operands()[0].RegClass);
894
895 // Split 64-bit constants into 32-bits for folding.
896 if (UseOp.getSubReg() && AMDGPU::getRegBitWidth(FoldRC->getID()) == 64) {
897 Register UseReg = UseOp.getReg();
898 const TargetRegisterClass *UseRC = MRI->getRegClass(UseReg);
899
900 if (AMDGPU::getRegBitWidth(UseRC->getID()) != 64)
901 return;
902
903 APInt Imm(64, OpToFold.getImm());
904 if (UseOp.getSubReg() == AMDGPU::sub0) {
905 Imm = Imm.getLoBits(32);
906 } else {
907 assert(UseOp.getSubReg() == AMDGPU::sub1);
908 Imm = Imm.getHiBits(32);
909 }
910
911 MachineOperand ImmOp = MachineOperand::CreateImm(Imm.getSExtValue());
912 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &ImmOp);
913 return;
914 }
915
916 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold);
917 }
918
evalBinaryInstruction(unsigned Opcode,int32_t & Result,uint32_t LHS,uint32_t RHS)919 static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
920 uint32_t LHS, uint32_t RHS) {
921 switch (Opcode) {
922 case AMDGPU::V_AND_B32_e64:
923 case AMDGPU::V_AND_B32_e32:
924 case AMDGPU::S_AND_B32:
925 Result = LHS & RHS;
926 return true;
927 case AMDGPU::V_OR_B32_e64:
928 case AMDGPU::V_OR_B32_e32:
929 case AMDGPU::S_OR_B32:
930 Result = LHS | RHS;
931 return true;
932 case AMDGPU::V_XOR_B32_e64:
933 case AMDGPU::V_XOR_B32_e32:
934 case AMDGPU::S_XOR_B32:
935 Result = LHS ^ RHS;
936 return true;
937 case AMDGPU::S_XNOR_B32:
938 Result = ~(LHS ^ RHS);
939 return true;
940 case AMDGPU::S_NAND_B32:
941 Result = ~(LHS & RHS);
942 return true;
943 case AMDGPU::S_NOR_B32:
944 Result = ~(LHS | RHS);
945 return true;
946 case AMDGPU::S_ANDN2_B32:
947 Result = LHS & ~RHS;
948 return true;
949 case AMDGPU::S_ORN2_B32:
950 Result = LHS | ~RHS;
951 return true;
952 case AMDGPU::V_LSHL_B32_e64:
953 case AMDGPU::V_LSHL_B32_e32:
954 case AMDGPU::S_LSHL_B32:
955 // The instruction ignores the high bits for out of bounds shifts.
956 Result = LHS << (RHS & 31);
957 return true;
958 case AMDGPU::V_LSHLREV_B32_e64:
959 case AMDGPU::V_LSHLREV_B32_e32:
960 Result = RHS << (LHS & 31);
961 return true;
962 case AMDGPU::V_LSHR_B32_e64:
963 case AMDGPU::V_LSHR_B32_e32:
964 case AMDGPU::S_LSHR_B32:
965 Result = LHS >> (RHS & 31);
966 return true;
967 case AMDGPU::V_LSHRREV_B32_e64:
968 case AMDGPU::V_LSHRREV_B32_e32:
969 Result = RHS >> (LHS & 31);
970 return true;
971 case AMDGPU::V_ASHR_I32_e64:
972 case AMDGPU::V_ASHR_I32_e32:
973 case AMDGPU::S_ASHR_I32:
974 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
975 return true;
976 case AMDGPU::V_ASHRREV_I32_e64:
977 case AMDGPU::V_ASHRREV_I32_e32:
978 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
979 return true;
980 default:
981 return false;
982 }
983 }
984
getMovOpc(bool IsScalar)985 static unsigned getMovOpc(bool IsScalar) {
986 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
987 }
988
mutateCopyOp(MachineInstr & MI,const MCInstrDesc & NewDesc)989 static void mutateCopyOp(MachineInstr &MI, const MCInstrDesc &NewDesc) {
990 MI.setDesc(NewDesc);
991
992 // Remove any leftover implicit operands from mutating the instruction. e.g.
993 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
994 // anymore.
995 const MCInstrDesc &Desc = MI.getDesc();
996 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
997 Desc.implicit_defs().size();
998
999 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
1000 MI.removeOperand(I);
1001 }
1002
1003 MachineOperand *
getImmOrMaterializedImm(MachineOperand & Op) const1004 SIFoldOperands::getImmOrMaterializedImm(MachineOperand &Op) const {
1005 // If this has a subregister, it obviously is a register source.
1006 if (!Op.isReg() || Op.getSubReg() != AMDGPU::NoSubRegister ||
1007 !Op.getReg().isVirtual())
1008 return &Op;
1009
1010 MachineInstr *Def = MRI->getVRegDef(Op.getReg());
1011 if (Def && Def->isMoveImmediate()) {
1012 MachineOperand &ImmSrc = Def->getOperand(1);
1013 if (ImmSrc.isImm())
1014 return &ImmSrc;
1015 }
1016
1017 return &Op;
1018 }
1019
1020 // Try to simplify operations with a constant that may appear after instruction
1021 // selection.
1022 // TODO: See if a frame index with a fixed offset can fold.
tryConstantFoldOp(MachineInstr * MI) const1023 bool SIFoldOperands::tryConstantFoldOp(MachineInstr *MI) const {
1024 unsigned Opc = MI->getOpcode();
1025
1026 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1027 if (Src0Idx == -1)
1028 return false;
1029 MachineOperand *Src0 = getImmOrMaterializedImm(MI->getOperand(Src0Idx));
1030
1031 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1032 Opc == AMDGPU::S_NOT_B32) &&
1033 Src0->isImm()) {
1034 MI->getOperand(1).ChangeToImmediate(~Src0->getImm());
1035 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32)));
1036 return true;
1037 }
1038
1039 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1040 if (Src1Idx == -1)
1041 return false;
1042 MachineOperand *Src1 = getImmOrMaterializedImm(MI->getOperand(Src1Idx));
1043
1044 if (!Src0->isImm() && !Src1->isImm())
1045 return false;
1046
1047 // and k0, k1 -> v_mov_b32 (k0 & k1)
1048 // or k0, k1 -> v_mov_b32 (k0 | k1)
1049 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1050 if (Src0->isImm() && Src1->isImm()) {
1051 int32_t NewImm;
1052 if (!evalBinaryInstruction(Opc, NewImm, Src0->getImm(), Src1->getImm()))
1053 return false;
1054
1055 bool IsSGPR = TRI->isSGPRReg(*MRI, MI->getOperand(0).getReg());
1056
1057 // Be careful to change the right operand, src0 may belong to a different
1058 // instruction.
1059 MI->getOperand(Src0Idx).ChangeToImmediate(NewImm);
1060 MI->removeOperand(Src1Idx);
1061 mutateCopyOp(*MI, TII->get(getMovOpc(IsSGPR)));
1062 return true;
1063 }
1064
1065 if (!MI->isCommutable())
1066 return false;
1067
1068 if (Src0->isImm() && !Src1->isImm()) {
1069 std::swap(Src0, Src1);
1070 std::swap(Src0Idx, Src1Idx);
1071 }
1072
1073 int32_t Src1Val = static_cast<int32_t>(Src1->getImm());
1074 if (Opc == AMDGPU::V_OR_B32_e64 ||
1075 Opc == AMDGPU::V_OR_B32_e32 ||
1076 Opc == AMDGPU::S_OR_B32) {
1077 if (Src1Val == 0) {
1078 // y = or x, 0 => y = copy x
1079 MI->removeOperand(Src1Idx);
1080 mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1081 } else if (Src1Val == -1) {
1082 // y = or x, -1 => y = v_mov_b32 -1
1083 MI->removeOperand(Src1Idx);
1084 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32)));
1085 } else
1086 return false;
1087
1088 return true;
1089 }
1090
1091 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1092 Opc == AMDGPU::S_AND_B32) {
1093 if (Src1Val == 0) {
1094 // y = and x, 0 => y = v_mov_b32 0
1095 MI->removeOperand(Src0Idx);
1096 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32)));
1097 } else if (Src1Val == -1) {
1098 // y = and x, -1 => y = copy x
1099 MI->removeOperand(Src1Idx);
1100 mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1101 } else
1102 return false;
1103
1104 return true;
1105 }
1106
1107 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1108 Opc == AMDGPU::S_XOR_B32) {
1109 if (Src1Val == 0) {
1110 // y = xor x, 0 => y = copy x
1111 MI->removeOperand(Src1Idx);
1112 mutateCopyOp(*MI, TII->get(AMDGPU::COPY));
1113 return true;
1114 }
1115 }
1116
1117 return false;
1118 }
1119
1120 // Try to fold an instruction into a simpler one
tryFoldCndMask(MachineInstr & MI) const1121 bool SIFoldOperands::tryFoldCndMask(MachineInstr &MI) const {
1122 unsigned Opc = MI.getOpcode();
1123 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1124 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
1125 return false;
1126
1127 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1128 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1129 if (!Src1->isIdenticalTo(*Src0)) {
1130 auto *Src0Imm = getImmOrMaterializedImm(*Src0);
1131 auto *Src1Imm = getImmOrMaterializedImm(*Src1);
1132 if (!Src1Imm->isIdenticalTo(*Src0Imm))
1133 return false;
1134 }
1135
1136 int Src1ModIdx =
1137 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers);
1138 int Src0ModIdx =
1139 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
1140 if ((Src1ModIdx != -1 && MI.getOperand(Src1ModIdx).getImm() != 0) ||
1141 (Src0ModIdx != -1 && MI.getOperand(Src0ModIdx).getImm() != 0))
1142 return false;
1143
1144 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1145 auto &NewDesc =
1146 TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false));
1147 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
1148 if (Src2Idx != -1)
1149 MI.removeOperand(Src2Idx);
1150 MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1));
1151 if (Src1ModIdx != -1)
1152 MI.removeOperand(Src1ModIdx);
1153 if (Src0ModIdx != -1)
1154 MI.removeOperand(Src0ModIdx);
1155 mutateCopyOp(MI, NewDesc);
1156 LLVM_DEBUG(dbgs() << MI);
1157 return true;
1158 }
1159
tryFoldZeroHighBits(MachineInstr & MI) const1160 bool SIFoldOperands::tryFoldZeroHighBits(MachineInstr &MI) const {
1161 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 &&
1162 MI.getOpcode() != AMDGPU::V_AND_B32_e32)
1163 return false;
1164
1165 MachineOperand *Src0 = getImmOrMaterializedImm(MI.getOperand(1));
1166 if (!Src0->isImm() || Src0->getImm() != 0xffff)
1167 return false;
1168
1169 Register Src1 = MI.getOperand(2).getReg();
1170 MachineInstr *SrcDef = MRI->getVRegDef(Src1);
1171 if (!ST->zeroesHigh16BitsOfDest(SrcDef->getOpcode()))
1172 return false;
1173
1174 Register Dst = MI.getOperand(0).getReg();
1175 MRI->replaceRegWith(Dst, SrcDef->getOperand(0).getReg());
1176 MI.eraseFromParent();
1177 return true;
1178 }
1179
foldInstOperand(MachineInstr & MI,MachineOperand & OpToFold) const1180 bool SIFoldOperands::foldInstOperand(MachineInstr &MI,
1181 MachineOperand &OpToFold) const {
1182 // We need mutate the operands of new mov instructions to add implicit
1183 // uses of EXEC, but adding them invalidates the use_iterator, so defer
1184 // this.
1185 SmallVector<MachineInstr *, 4> CopiesToReplace;
1186 SmallVector<FoldCandidate, 4> FoldList;
1187 MachineOperand &Dst = MI.getOperand(0);
1188 bool Changed = false;
1189
1190 if (OpToFold.isImm()) {
1191 for (auto &UseMI :
1192 make_early_inc_range(MRI->use_nodbg_instructions(Dst.getReg()))) {
1193 // Folding the immediate may reveal operations that can be constant
1194 // folded or replaced with a copy. This can happen for example after
1195 // frame indices are lowered to constants or from splitting 64-bit
1196 // constants.
1197 //
1198 // We may also encounter cases where one or both operands are
1199 // immediates materialized into a register, which would ordinarily not
1200 // be folded due to multiple uses or operand constraints.
1201 if (tryConstantFoldOp(&UseMI)) {
1202 LLVM_DEBUG(dbgs() << "Constant folded " << UseMI);
1203 Changed = true;
1204 }
1205 }
1206 }
1207
1208 SmallVector<MachineOperand *, 4> UsesToProcess;
1209 for (auto &Use : MRI->use_nodbg_operands(Dst.getReg()))
1210 UsesToProcess.push_back(&Use);
1211 for (auto *U : UsesToProcess) {
1212 MachineInstr *UseMI = U->getParent();
1213 foldOperand(OpToFold, UseMI, UseMI->getOperandNo(U), FoldList,
1214 CopiesToReplace);
1215 }
1216
1217 if (CopiesToReplace.empty() && FoldList.empty())
1218 return Changed;
1219
1220 MachineFunction *MF = MI.getParent()->getParent();
1221 // Make sure we add EXEC uses to any new v_mov instructions created.
1222 for (MachineInstr *Copy : CopiesToReplace)
1223 Copy->addImplicitDefUseOperands(*MF);
1224
1225 for (FoldCandidate &Fold : FoldList) {
1226 assert(!Fold.isReg() || Fold.OpToFold);
1227 if (Fold.isReg() && Fold.OpToFold->getReg().isVirtual()) {
1228 Register Reg = Fold.OpToFold->getReg();
1229 MachineInstr *DefMI = Fold.OpToFold->getParent();
1230 if (DefMI->readsRegister(AMDGPU::EXEC, TRI) &&
1231 execMayBeModifiedBeforeUse(*MRI, Reg, *DefMI, *Fold.UseMI))
1232 continue;
1233 }
1234 if (updateOperand(Fold)) {
1235 // Clear kill flags.
1236 if (Fold.isReg()) {
1237 assert(Fold.OpToFold && Fold.OpToFold->isReg());
1238 // FIXME: Probably shouldn't bother trying to fold if not an
1239 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
1240 // copies.
1241 MRI->clearKillFlags(Fold.OpToFold->getReg());
1242 }
1243 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
1244 << static_cast<int>(Fold.UseOpNo) << " of "
1245 << *Fold.UseMI);
1246 } else if (Fold.Commuted) {
1247 // Restoring instruction's original operand order if fold has failed.
1248 TII->commuteInstruction(*Fold.UseMI, false);
1249 }
1250 }
1251 return true;
1252 }
1253
tryFoldFoldableCopy(MachineInstr & MI,MachineOperand * & CurrentKnownM0Val) const1254 bool SIFoldOperands::tryFoldFoldableCopy(
1255 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
1256 // Specially track simple redefs of m0 to the same value in a block, so we
1257 // can erase the later ones.
1258 if (MI.getOperand(0).getReg() == AMDGPU::M0) {
1259 MachineOperand &NewM0Val = MI.getOperand(1);
1260 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(NewM0Val)) {
1261 MI.eraseFromParent();
1262 return true;
1263 }
1264
1265 // We aren't tracking other physical registers
1266 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
1267 ? nullptr
1268 : &NewM0Val;
1269 return false;
1270 }
1271
1272 MachineOperand &OpToFold = MI.getOperand(1);
1273 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1274
1275 // FIXME: We could also be folding things like TargetIndexes.
1276 if (!FoldingImm && !OpToFold.isReg())
1277 return false;
1278
1279 if (OpToFold.isReg() && !OpToFold.getReg().isVirtual())
1280 return false;
1281
1282 // Prevent folding operands backwards in the function. For example,
1283 // the COPY opcode must not be replaced by 1 in this example:
1284 //
1285 // %3 = COPY %vgpr0; VGPR_32:%3
1286 // ...
1287 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
1288 if (!MI.getOperand(0).getReg().isVirtual())
1289 return false;
1290
1291 bool Changed = foldInstOperand(MI, OpToFold);
1292
1293 // If we managed to fold all uses of this copy then we might as well
1294 // delete it now.
1295 // The only reason we need to follow chains of copies here is that
1296 // tryFoldRegSequence looks forward through copies before folding a
1297 // REG_SEQUENCE into its eventual users.
1298 auto *InstToErase = &MI;
1299 while (MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) {
1300 auto &SrcOp = InstToErase->getOperand(1);
1301 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
1302 InstToErase->eraseFromParent();
1303 Changed = true;
1304 InstToErase = nullptr;
1305 if (!SrcReg || SrcReg.isPhysical())
1306 break;
1307 InstToErase = MRI->getVRegDef(SrcReg);
1308 if (!InstToErase || !TII->isFoldableCopy(*InstToErase))
1309 break;
1310 }
1311
1312 if (InstToErase && InstToErase->isRegSequence() &&
1313 MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) {
1314 InstToErase->eraseFromParent();
1315 Changed = true;
1316 }
1317
1318 return Changed;
1319 }
1320
1321 // Clamp patterns are canonically selected to v_max_* instructions, so only
1322 // handle them.
isClamp(const MachineInstr & MI) const1323 const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const {
1324 unsigned Op = MI.getOpcode();
1325 switch (Op) {
1326 case AMDGPU::V_MAX_F32_e64:
1327 case AMDGPU::V_MAX_F16_e64:
1328 case AMDGPU::V_MAX_F16_t16_e64:
1329 case AMDGPU::V_MAX_F64_e64:
1330 case AMDGPU::V_PK_MAX_F16: {
1331 if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm())
1332 return nullptr;
1333
1334 // Make sure sources are identical.
1335 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1336 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1337 if (!Src0->isReg() || !Src1->isReg() ||
1338 Src0->getReg() != Src1->getReg() ||
1339 Src0->getSubReg() != Src1->getSubReg() ||
1340 Src0->getSubReg() != AMDGPU::NoSubRegister)
1341 return nullptr;
1342
1343 // Can't fold up if we have modifiers.
1344 if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
1345 return nullptr;
1346
1347 unsigned Src0Mods
1348 = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm();
1349 unsigned Src1Mods
1350 = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm();
1351
1352 // Having a 0 op_sel_hi would require swizzling the output in the source
1353 // instruction, which we can't do.
1354 unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1
1355 : 0u;
1356 if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
1357 return nullptr;
1358 return Src0;
1359 }
1360 default:
1361 return nullptr;
1362 }
1363 }
1364
1365 // FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
tryFoldClamp(MachineInstr & MI)1366 bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) {
1367 const MachineOperand *ClampSrc = isClamp(MI);
1368 if (!ClampSrc || !MRI->hasOneNonDBGUser(ClampSrc->getReg()))
1369 return false;
1370
1371 MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg());
1372
1373 // The type of clamp must be compatible.
1374 if (TII->getClampMask(*Def) != TII->getClampMask(MI))
1375 return false;
1376
1377 MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp);
1378 if (!DefClamp)
1379 return false;
1380
1381 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
1382
1383 // Clamp is applied after omod, so it is OK if omod is set.
1384 DefClamp->setImm(1);
1385 MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
1386 MI.eraseFromParent();
1387
1388 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
1389 // instruction, so we might as well convert it to the more flexible VOP3-only
1390 // mad/fma form.
1391 if (TII->convertToThreeAddress(*Def, nullptr, nullptr))
1392 Def->eraseFromParent();
1393
1394 return true;
1395 }
1396
getOModValue(unsigned Opc,int64_t Val)1397 static int getOModValue(unsigned Opc, int64_t Val) {
1398 switch (Opc) {
1399 case AMDGPU::V_MUL_F64_e64: {
1400 switch (Val) {
1401 case 0x3fe0000000000000: // 0.5
1402 return SIOutMods::DIV2;
1403 case 0x4000000000000000: // 2.0
1404 return SIOutMods::MUL2;
1405 case 0x4010000000000000: // 4.0
1406 return SIOutMods::MUL4;
1407 default:
1408 return SIOutMods::NONE;
1409 }
1410 }
1411 case AMDGPU::V_MUL_F32_e64: {
1412 switch (static_cast<uint32_t>(Val)) {
1413 case 0x3f000000: // 0.5
1414 return SIOutMods::DIV2;
1415 case 0x40000000: // 2.0
1416 return SIOutMods::MUL2;
1417 case 0x40800000: // 4.0
1418 return SIOutMods::MUL4;
1419 default:
1420 return SIOutMods::NONE;
1421 }
1422 }
1423 case AMDGPU::V_MUL_F16_e64:
1424 case AMDGPU::V_MUL_F16_t16_e64: {
1425 switch (static_cast<uint16_t>(Val)) {
1426 case 0x3800: // 0.5
1427 return SIOutMods::DIV2;
1428 case 0x4000: // 2.0
1429 return SIOutMods::MUL2;
1430 case 0x4400: // 4.0
1431 return SIOutMods::MUL4;
1432 default:
1433 return SIOutMods::NONE;
1434 }
1435 }
1436 default:
1437 llvm_unreachable("invalid mul opcode");
1438 }
1439 }
1440
1441 // FIXME: Does this really not support denormals with f16?
1442 // FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
1443 // handled, so will anything other than that break?
1444 std::pair<const MachineOperand *, int>
isOMod(const MachineInstr & MI) const1445 SIFoldOperands::isOMod(const MachineInstr &MI) const {
1446 unsigned Op = MI.getOpcode();
1447 switch (Op) {
1448 case AMDGPU::V_MUL_F64_e64:
1449 case AMDGPU::V_MUL_F32_e64:
1450 case AMDGPU::V_MUL_F16_t16_e64:
1451 case AMDGPU::V_MUL_F16_e64: {
1452 // If output denormals are enabled, omod is ignored.
1453 if ((Op == AMDGPU::V_MUL_F32_e64 &&
1454 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
1455 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F16_e64 ||
1456 Op == AMDGPU::V_MUL_F16_t16_e64) &&
1457 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
1458 return std::pair(nullptr, SIOutMods::NONE);
1459
1460 const MachineOperand *RegOp = nullptr;
1461 const MachineOperand *ImmOp = nullptr;
1462 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1463 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1464 if (Src0->isImm()) {
1465 ImmOp = Src0;
1466 RegOp = Src1;
1467 } else if (Src1->isImm()) {
1468 ImmOp = Src1;
1469 RegOp = Src0;
1470 } else
1471 return std::pair(nullptr, SIOutMods::NONE);
1472
1473 int OMod = getOModValue(Op, ImmOp->getImm());
1474 if (OMod == SIOutMods::NONE ||
1475 TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
1476 TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
1477 TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
1478 TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
1479 return std::pair(nullptr, SIOutMods::NONE);
1480
1481 return std::pair(RegOp, OMod);
1482 }
1483 case AMDGPU::V_ADD_F64_e64:
1484 case AMDGPU::V_ADD_F32_e64:
1485 case AMDGPU::V_ADD_F16_e64:
1486 case AMDGPU::V_ADD_F16_t16_e64: {
1487 // If output denormals are enabled, omod is ignored.
1488 if ((Op == AMDGPU::V_ADD_F32_e64 &&
1489 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
1490 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F16_e64 ||
1491 Op == AMDGPU::V_ADD_F16_t16_e64) &&
1492 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
1493 return std::pair(nullptr, SIOutMods::NONE);
1494
1495 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
1496 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
1497 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
1498
1499 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
1500 Src0->getSubReg() == Src1->getSubReg() &&
1501 !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) &&
1502 !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) &&
1503 !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) &&
1504 !TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
1505 return std::pair(Src0, SIOutMods::MUL2);
1506
1507 return std::pair(nullptr, SIOutMods::NONE);
1508 }
1509 default:
1510 return std::pair(nullptr, SIOutMods::NONE);
1511 }
1512 }
1513
1514 // FIXME: Does this need to check IEEE bit on function?
tryFoldOMod(MachineInstr & MI)1515 bool SIFoldOperands::tryFoldOMod(MachineInstr &MI) {
1516 const MachineOperand *RegOp;
1517 int OMod;
1518 std::tie(RegOp, OMod) = isOMod(MI);
1519 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
1520 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
1521 !MRI->hasOneNonDBGUser(RegOp->getReg()))
1522 return false;
1523
1524 MachineInstr *Def = MRI->getVRegDef(RegOp->getReg());
1525 MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod);
1526 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
1527 return false;
1528
1529 // Clamp is applied after omod. If the source already has clamp set, don't
1530 // fold it.
1531 if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp))
1532 return false;
1533
1534 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
1535
1536 DefOMod->setImm(OMod);
1537 MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
1538 MI.eraseFromParent();
1539
1540 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
1541 // instruction, so we might as well convert it to the more flexible VOP3-only
1542 // mad/fma form.
1543 if (TII->convertToThreeAddress(*Def, nullptr, nullptr))
1544 Def->eraseFromParent();
1545
1546 return true;
1547 }
1548
1549 // Try to fold a reg_sequence with vgpr output and agpr inputs into an
1550 // instruction which can take an agpr. So far that means a store.
tryFoldRegSequence(MachineInstr & MI)1551 bool SIFoldOperands::tryFoldRegSequence(MachineInstr &MI) {
1552 assert(MI.isRegSequence());
1553 auto Reg = MI.getOperand(0).getReg();
1554
1555 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(*MRI, Reg) ||
1556 !MRI->hasOneNonDBGUse(Reg))
1557 return false;
1558
1559 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
1560 if (!getRegSeqInit(Defs, Reg, MCOI::OPERAND_REGISTER))
1561 return false;
1562
1563 for (auto &Def : Defs) {
1564 const auto *Op = Def.first;
1565 if (!Op->isReg())
1566 return false;
1567 if (TRI->isAGPR(*MRI, Op->getReg()))
1568 continue;
1569 // Maybe this is a COPY from AREG
1570 const MachineInstr *SubDef = MRI->getVRegDef(Op->getReg());
1571 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(1).getSubReg())
1572 return false;
1573 if (!TRI->isAGPR(*MRI, SubDef->getOperand(1).getReg()))
1574 return false;
1575 }
1576
1577 MachineOperand *Op = &*MRI->use_nodbg_begin(Reg);
1578 MachineInstr *UseMI = Op->getParent();
1579 while (UseMI->isCopy() && !Op->getSubReg()) {
1580 Reg = UseMI->getOperand(0).getReg();
1581 if (!TRI->isVGPR(*MRI, Reg) || !MRI->hasOneNonDBGUse(Reg))
1582 return false;
1583 Op = &*MRI->use_nodbg_begin(Reg);
1584 UseMI = Op->getParent();
1585 }
1586
1587 if (Op->getSubReg())
1588 return false;
1589
1590 unsigned OpIdx = Op - &UseMI->getOperand(0);
1591 const MCInstrDesc &InstDesc = UseMI->getDesc();
1592 const TargetRegisterClass *OpRC =
1593 TII->getRegClass(InstDesc, OpIdx, TRI, *MI.getMF());
1594 if (!OpRC || !TRI->isVectorSuperClass(OpRC))
1595 return false;
1596
1597 const auto *NewDstRC = TRI->getEquivalentAGPRClass(MRI->getRegClass(Reg));
1598 auto Dst = MRI->createVirtualRegister(NewDstRC);
1599 auto RS = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1600 TII->get(AMDGPU::REG_SEQUENCE), Dst);
1601
1602 for (unsigned I = 0; I < Defs.size(); ++I) {
1603 MachineOperand *Def = Defs[I].first;
1604 Def->setIsKill(false);
1605 if (TRI->isAGPR(*MRI, Def->getReg())) {
1606 RS.add(*Def);
1607 } else { // This is a copy
1608 MachineInstr *SubDef = MRI->getVRegDef(Def->getReg());
1609 SubDef->getOperand(1).setIsKill(false);
1610 RS.addReg(SubDef->getOperand(1).getReg(), 0, Def->getSubReg());
1611 }
1612 RS.addImm(Defs[I].second);
1613 }
1614
1615 Op->setReg(Dst);
1616 if (!TII->isOperandLegal(*UseMI, OpIdx, Op)) {
1617 Op->setReg(Reg);
1618 RS->eraseFromParent();
1619 return false;
1620 }
1621
1622 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
1623
1624 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
1625 // in which case we can erase them all later in runOnMachineFunction.
1626 if (MRI->use_nodbg_empty(MI.getOperand(0).getReg()))
1627 MI.eraseFromParent();
1628 return true;
1629 }
1630
1631 // Try to hoist an AGPR to VGPR copy out of the loop across a LCSSA PHI.
1632 // This should allow folding of an AGPR into a consumer which may support it.
1633 // I.e.:
1634 //
1635 // loop: // loop:
1636 // %1:vreg = COPY %0:areg // exit:
1637 // exit: => // %1:areg = PHI %0:areg, %loop
1638 // %2:vreg = PHI %1:vreg, %loop // %2:vreg = COPY %1:areg
tryFoldLCSSAPhi(MachineInstr & PHI)1639 bool SIFoldOperands::tryFoldLCSSAPhi(MachineInstr &PHI) {
1640 assert(PHI.isPHI());
1641
1642 if (PHI.getNumExplicitOperands() != 3) // Single input LCSSA PHI
1643 return false;
1644
1645 Register PhiIn = PHI.getOperand(1).getReg();
1646 Register PhiOut = PHI.getOperand(0).getReg();
1647 if (PHI.getOperand(1).getSubReg() ||
1648 !TRI->isVGPR(*MRI, PhiIn) || !TRI->isVGPR(*MRI, PhiOut))
1649 return false;
1650
1651 // A single use should not matter for correctness, but if it has another use
1652 // inside the loop we may perform copy twice in a worst case.
1653 if (!MRI->hasOneNonDBGUse(PhiIn))
1654 return false;
1655
1656 MachineInstr *Copy = MRI->getVRegDef(PhiIn);
1657 if (!Copy || !Copy->isCopy())
1658 return false;
1659
1660 Register CopyIn = Copy->getOperand(1).getReg();
1661 if (!TRI->isAGPR(*MRI, CopyIn) || Copy->getOperand(1).getSubReg())
1662 return false;
1663
1664 const TargetRegisterClass *ARC = MRI->getRegClass(CopyIn);
1665 Register NewReg = MRI->createVirtualRegister(ARC);
1666 PHI.getOperand(1).setReg(CopyIn);
1667 PHI.getOperand(0).setReg(NewReg);
1668
1669 MachineBasicBlock *MBB = PHI.getParent();
1670 BuildMI(*MBB, MBB->getFirstNonPHI(), Copy->getDebugLoc(),
1671 TII->get(AMDGPU::COPY), PhiOut)
1672 .addReg(NewReg, RegState::Kill);
1673 Copy->eraseFromParent(); // We know this copy had a single use.
1674
1675 LLVM_DEBUG(dbgs() << "Folded " << PHI);
1676
1677 return true;
1678 }
1679
1680 // Attempt to convert VGPR load to an AGPR load.
tryFoldLoad(MachineInstr & MI)1681 bool SIFoldOperands::tryFoldLoad(MachineInstr &MI) {
1682 assert(MI.mayLoad());
1683 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
1684 return false;
1685
1686 MachineOperand &Def = MI.getOperand(0);
1687 if (!Def.isDef())
1688 return false;
1689
1690 Register DefReg = Def.getReg();
1691
1692 if (DefReg.isPhysical() || !TRI->isVGPR(*MRI, DefReg))
1693 return false;
1694
1695 SmallVector<const MachineInstr*, 8> Users;
1696 SmallVector<Register, 8> MoveRegs;
1697 for (const MachineInstr &I : MRI->use_nodbg_instructions(DefReg))
1698 Users.push_back(&I);
1699
1700 if (Users.empty())
1701 return false;
1702
1703 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
1704 while (!Users.empty()) {
1705 const MachineInstr *I = Users.pop_back_val();
1706 if (!I->isCopy() && !I->isRegSequence())
1707 return false;
1708 Register DstReg = I->getOperand(0).getReg();
1709 // Physical registers may have more than one instruction definitions
1710 if (DstReg.isPhysical())
1711 return false;
1712 if (TRI->isAGPR(*MRI, DstReg))
1713 continue;
1714 MoveRegs.push_back(DstReg);
1715 for (const MachineInstr &U : MRI->use_nodbg_instructions(DstReg))
1716 Users.push_back(&U);
1717 }
1718
1719 const TargetRegisterClass *RC = MRI->getRegClass(DefReg);
1720 MRI->setRegClass(DefReg, TRI->getEquivalentAGPRClass(RC));
1721 if (!TII->isOperandLegal(MI, 0, &Def)) {
1722 MRI->setRegClass(DefReg, RC);
1723 return false;
1724 }
1725
1726 while (!MoveRegs.empty()) {
1727 Register Reg = MoveRegs.pop_back_val();
1728 MRI->setRegClass(Reg, TRI->getEquivalentAGPRClass(MRI->getRegClass(Reg)));
1729 }
1730
1731 LLVM_DEBUG(dbgs() << "Folded " << MI);
1732
1733 return true;
1734 }
1735
runOnMachineFunction(MachineFunction & MF)1736 bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
1737 if (skipFunction(MF.getFunction()))
1738 return false;
1739
1740 MRI = &MF.getRegInfo();
1741 ST = &MF.getSubtarget<GCNSubtarget>();
1742 TII = ST->getInstrInfo();
1743 TRI = &TII->getRegisterInfo();
1744 MFI = MF.getInfo<SIMachineFunctionInfo>();
1745
1746 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
1747 // correctly handle signed zeros.
1748 //
1749 // FIXME: Also need to check strictfp
1750 bool IsIEEEMode = MFI->getMode().IEEE;
1751 bool HasNSZ = MFI->hasNoSignedZerosFPMath();
1752
1753 bool Changed = false;
1754 for (MachineBasicBlock *MBB : depth_first(&MF)) {
1755 MachineOperand *CurrentKnownM0Val = nullptr;
1756 for (auto &MI : make_early_inc_range(*MBB)) {
1757 Changed |= tryFoldCndMask(MI);
1758
1759 if (tryFoldZeroHighBits(MI)) {
1760 Changed = true;
1761 continue;
1762 }
1763
1764 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
1765 Changed = true;
1766 continue;
1767 }
1768
1769 if (MI.isPHI() && tryFoldLCSSAPhi(MI)) {
1770 Changed = true;
1771 continue;
1772 }
1773
1774 if (MI.mayLoad() && tryFoldLoad(MI)) {
1775 Changed = true;
1776 continue;
1777 }
1778
1779 if (TII->isFoldableCopy(MI)) {
1780 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
1781 continue;
1782 }
1783
1784 // Saw an unknown clobber of m0, so we no longer know what it is.
1785 if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI))
1786 CurrentKnownM0Val = nullptr;
1787
1788 // TODO: Omod might be OK if there is NSZ only on the source
1789 // instruction, and not the omod multiply.
1790 if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) ||
1791 !tryFoldOMod(MI))
1792 Changed |= tryFoldClamp(MI);
1793 }
1794 }
1795
1796 return Changed;
1797 }
1798