xref: /llvm-project/llvm/lib/Target/AMDGPU/VOPInstructions.td (revision 5d1c596ab47b9412bb36bdfb0520d9af1343a5ce)
1//===-- VOPInstructions.td - Vector Instruction Definitions ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9// dummies for outer let
10class LetDummies {
11  bit TRANS;
12  bit ReadsModeReg;
13  bit mayRaiseFPException;
14  bit isCommutable;
15  bit isConvertibleToThreeAddress;
16  bit isMoveImm;
17  bit isReMaterializable;
18  bit isAsCheapAsAMove;
19  bit FPDPRounding;
20  Predicate SubtargetPredicate;
21  string Constraints;
22  string DisableEncoding;
23  list<SchedReadWrite> SchedRW;
24  list<Register> Uses;
25  list<Register> Defs;
26  list<Predicate> OtherPredicates;
27  Predicate AssemblerPredicate;
28  string DecoderNamespace;
29}
30
31class VOP <string opName> {
32  string OpName = opName;
33}
34
35// First 13 insts from VOPDY are also VOPDX. DOT2ACC_F32_BF16 is omitted
36defvar VOPDX_Max_Index = 12;
37
38class VOPD_Component<bits<5> OpIn, string vOPDName> {
39  Instruction BaseVOP = !cast<Instruction>(NAME);
40  string VOPDName = "v_dual_" # !substr(vOPDName, 2);
41  bits<5> VOPDOp = OpIn;
42  bit CanBeVOPDX = !le(VOPDOp, VOPDX_Max_Index);
43}
44
45class VOPAnyCommon <dag outs, dag ins, string asm, list<dag> pattern> :
46    InstSI <outs, ins, asm, pattern> {
47
48  let mayLoad = 0;
49  let mayStore = 0;
50  let hasSideEffects = 0;
51  let UseNamedOperandTable = 1;
52  let VALU = 1;
53  let Uses = !if(ReadsModeReg, [MODE, EXEC], [EXEC]);
54}
55
56class VOP_Pseudo <string opName, string suffix, VOPProfile P, dag outs, dag ins,
57                  string asm, list<dag> pattern> :
58  InstSI <outs, ins, asm, pattern>,
59  VOP <opName>,
60  SIMCInstr <opName#suffix, SIEncodingFamily.NONE> {
61  let isPseudo = 1;
62  let isCodeGenOnly = 1;
63  let UseNamedOperandTable = 1;
64
65  string Mnemonic = opName;
66  Instruction Opcode = !cast<Instruction>(NAME);
67  bit IsTrue16 = P.IsTrue16;
68  VOPProfile Pfl = P;
69
70  string AsmOperands;
71}
72
73class VOP3Common <dag outs, dag ins, string asm = "",
74                  list<dag> pattern = [], bit HasMods = 0> :
75  VOPAnyCommon <outs, ins, asm, pattern> {
76
77  // Using complex patterns gives VOP3 patterns a very high complexity rating,
78  // but standalone patterns are almost always preferred, so we need to adjust the
79  // priority lower.  The goal is to use a high number to reduce complexity to
80  // zero (or less than zero).
81  let AddedComplexity = -1000;
82
83  let VOP3 = 1;
84
85  let AsmVariantName = AMDGPUAsmVariants.VOP3;
86  let AsmMatchConverter = !if(HasMods, "cvtVOP3", "");
87
88  let isCodeGenOnly = 0;
89
90  int Size = 8;
91
92  // Because SGPRs may be allowed if there are multiple operands, we
93  // need a post-isel hook to insert copies in order to avoid
94  // violating constant bus requirements.
95  let hasPostISelHook = 1;
96}
97
98class VOP3_Pseudo <string opName, VOPProfile P, list<dag> pattern = [],
99                   bit isVOP3P = 0, bit isVop3OpSel = 0> :
100  VOP_Pseudo <opName, "_e64", P, P.Outs64,
101              !if(isVop3OpSel,
102                  P.InsVOP3OpSel,
103                  !if(!and(isVOP3P, P.IsPacked), P.InsVOP3P, P.Ins64)),
104              "", pattern> {
105
106  let VOP3_OPSEL = isVop3OpSel;
107  let IsPacked = P.IsPacked;
108  let IsMAI = P.IsMAI;
109  let IsWMMA = P.IsWMMA;
110  let IsSWMMAC = P.IsSWMMAC;
111
112  bit HasFP8DstByteSel = P.HasFP8DstByteSel;
113  bit HasFP4DstByteSel = P.HasFP4DstByteSel;
114
115  let AsmOperands = !if(!and(!not(P.IsTrue16), isVop3OpSel),
116                        P.AsmVOP3OpSel,
117                        !if(!and(isVOP3P, P.IsPacked), P.AsmVOP3P, P.Asm64));
118
119  let Size = 8;
120  let mayLoad = 0;
121  let mayStore = 0;
122  let hasSideEffects = 0;
123
124  // Because SGPRs may be allowed if there are multiple operands, we
125  // need a post-isel hook to insert copies in order to avoid
126  // violating constant bus requirements.
127  let hasPostISelHook = 1;
128
129  // Using complex patterns gives VOP3 patterns a very high complexity rating,
130  // but standalone patterns are almost always preferred, so we need to adjust the
131  // priority lower.  The goal is to use a high number to reduce complexity to
132  // zero (or less than zero).
133  let AddedComplexity = -1000;
134
135  let VOP3 = 1;
136  let VALU = 1;
137  let FPClamp = P.HasFPClamp;
138  let IntClamp = P.HasIntClamp;
139  let ClampLo = P.HasClampLo;
140  let ClampHi = P.HasClampHi;
141
142  let ReadsModeReg = !or(P.DstVT.isFP, P.Src0VT.isFP);
143
144  let mayRaiseFPException = ReadsModeReg;
145  let Uses = !if(ReadsModeReg, [MODE, EXEC], [EXEC]);
146
147  let AsmVariantName = AMDGPUAsmVariants.VOP3;
148  let AsmMatchConverter =
149    !if(isVOP3P,
150        "cvtVOP3P",
151        !if(!or(P.HasModifiers, P.HasOMod, P.HasClamp),
152            "cvtVOP3",
153            ""));
154}
155
156class VOP3P_Pseudo <string opName, VOPProfile P, list<dag> pattern = []> :
157  VOP3_Pseudo<opName, P, pattern, 1> {
158  let VOP3P = 1;
159  let IsDOT = P.IsDOT;
160}
161
162class VOP_Real<VOP_Pseudo ps> {
163  Instruction Opcode = !cast<Instruction>(NAME);
164  bit IsSingle = ps.Pfl.IsSingle;
165}
166
167class VOP3_Real <VOP_Pseudo ps, int EncodingFamily, string asm_name = ps.Mnemonic> :
168  VOP_Real <ps>,
169  InstSI <ps.OutOperandList, ps.InOperandList, asm_name # ps.AsmOperands, []>,
170  SIMCInstr <ps.PseudoInstr, EncodingFamily> {
171
172  let VALU = 1;
173  let VOP3 = 1;
174  let isPseudo = 0;
175  let isCodeGenOnly = 0;
176  let UseNamedOperandTable = 1;
177
178  // copy relevant pseudo op flags
179  let SubtargetPredicate = ps.SubtargetPredicate;
180  let WaveSizePredicate  = ps.WaveSizePredicate;
181  let OtherPredicates    = ps.OtherPredicates;
182  let True16Predicate    = ps.True16Predicate;
183  let AsmMatchConverter  = ps.AsmMatchConverter;
184  let AsmVariantName     = ps.AsmVariantName;
185  let Constraints        = ps.Constraints;
186  let DisableEncoding    = ps.DisableEncoding;
187  let TSFlags            = ps.TSFlags;
188  let UseNamedOperandTable = ps.UseNamedOperandTable;
189  let Uses                 = ps.Uses;
190  let Defs                 = ps.Defs;
191  let SchedRW              = ps.SchedRW;
192  let mayLoad              = ps.mayLoad;
193  let mayStore             = ps.mayStore;
194  let TRANS                = ps.TRANS;
195  let isConvergent         = ps.isConvergent;
196
197  VOPProfile Pfl = ps.Pfl;
198}
199
200class VOP3_Real_Gen <VOP_Pseudo ps, GFXGen Gen, string asm_name = ps.Mnemonic> :
201  VOP3_Real <ps, Gen.Subtarget, asm_name> {
202  let AssemblerPredicate = Gen.AssemblerPredicate;
203  let True16Predicate = !if(ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate);
204  let DecoderNamespace = Gen.DecoderNamespace#
205                         !if(ps.Pfl.IsRealTrue16, "", "_FAKE16");
206}
207
208// XXX - Is there any reason to distinguish this from regular VOP3
209// here?
210class VOP3P_Real<VOP_Pseudo ps, int EncodingFamily, string asm_name = ps.Mnemonic> :
211  VOP3_Real<ps, EncodingFamily, asm_name> {
212
213  // The v_wmma pseudos have extra constraints that we do not want to impose on the real instruction.
214  let Constraints        = !if(!eq(!substr(ps.Mnemonic,0,6), "v_wmma"), "", ps.Constraints);
215}
216
217class VOP3P_Real_Gen<VOP_Pseudo ps, GFXGen Gen, string asm_name = ps.Mnemonic> :
218  VOP3P_Real<ps, Gen.Subtarget, asm_name> {
219  let AssemblerPredicate = Gen.AssemblerPredicate;
220  let DecoderNamespace = Gen.DecoderNamespace;
221}
222
223class VOP3a<VOPProfile P> : Enc64 {
224  bits<4> src0_modifiers;
225  bits<9> src0;
226  bits<3> src1_modifiers;
227  bits<9> src1;
228  bits<3> src2_modifiers;
229  bits<9> src2;
230  bits<1> clamp;
231  bits<2> omod;
232
233  let Inst{8}     = !if(P.HasSrc0Mods, src0_modifiers{1}, 0);
234  let Inst{9}     = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
235  let Inst{10}    = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
236
237  let Inst{31-26} = 0x34; //encoding
238  let Inst{40-32} = !if(P.HasSrc0, src0, 0);
239  let Inst{49-41} = !if(P.HasSrc1, src1, 0);
240  let Inst{58-50} = !if(P.HasSrc2, src2, 0);
241  let Inst{60-59} = !if(P.HasOMod, omod, 0);
242  let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0);
243  let Inst{62}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0);
244  let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
245}
246
247// To avoid having different version of every type of operand depending on if
248// they are part of a True16 instruction or not, the operand encoding should be
249// the same for SGPR, imm, and VGPR_32 whether the instruction is True16 or not.
250class VOP3a_t16<VOPProfile P> : Enc64 {
251  bits<11> vdst;
252  bits<4> src0_modifiers;
253  bits<11> src0;
254  bits<3> src1_modifiers;
255  bits<11> src1;
256  bits<3> src2_modifiers;
257  bits<11> src2;
258  bits<1> clamp;
259  bits<2> omod;
260
261  let Inst{7-0} = !if(P.EmitDst, vdst{7-0}, 0);
262  let Inst{8}     = !if(P.HasSrc0Mods, src0_modifiers{1}, 0);
263  let Inst{9}     = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
264  let Inst{10}    = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
265  // 16-bit select fields which can be interpreted as OpSel or hi/lo suffix
266  let Inst{11} = !if(P.HasSrc0Mods, src0_modifiers{2}, 0);
267  let Inst{12} = !if(P.HasSrc1Mods, src1_modifiers{2}, 0);
268  let Inst{13} = !if(P.HasSrc2Mods, src2_modifiers{2}, 0);
269  let Inst{14} = !if(!and(P.HasDst, P.HasSrc0Mods), src0_modifiers{3}, 0);
270  let Inst{15} = !if(P.HasClamp, clamp{0}, 0);
271
272  let Inst{31-26} = 0x35;
273  let Inst{40-32} = !if(P.HasSrc0, src0{8-0}, 0);
274  let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
275  let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
276  let Inst{60-59} = !if(P.HasOMod, omod, 0);
277  let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0);
278  let Inst{62}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0);
279  let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
280}
281
282class VOP3a_gfx6_gfx7<bits<9> op, VOPProfile p> : VOP3a<p> {
283  let Inst{11}    = !if(p.HasClamp, clamp{0}, 0);
284  let Inst{25-17} = op;
285}
286
287class VOP3a_gfx10<bits<10> op, VOPProfile p> : VOP3a<p> {
288  let Inst{15}    = !if(p.HasClamp, clamp{0}, 0);
289  let Inst{25-16} = op;
290  let Inst{31-26} = 0x35;
291}
292
293class VOP3a_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3a_gfx10<op, p>;
294
295class VOP3a_vi <bits<10> op, VOPProfile P> : VOP3a<P> {
296  let Inst{25-16} = op;
297  let Inst{15}    = !if(P.HasClamp, clamp{0}, 0);
298}
299
300class VOP3e_gfx6_gfx7<bits<9> op, VOPProfile p> : VOP3a_gfx6_gfx7<op, p> {
301  bits<8> vdst;
302  let Inst{7-0} = !if(p.EmitDst, vdst{7-0}, 0);
303}
304
305class VOP3e_gfx10<bits<10> op, VOPProfile p> : VOP3a_gfx10<op, p> {
306  bits<8> vdst;
307  let Inst{7-0} = !if(p.EmitDst, vdst{7-0}, 0);
308}
309
310class VOP3e_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p>;
311
312class VOP3e_t16_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3a_t16<p> {
313  let Inst{25-16} = op;
314}
315
316class VOP3e_vi <bits<10> op, VOPProfile P> : VOP3a_vi <op, P> {
317  bits<8> vdst;
318  let Inst{7-0} = !if(P.EmitDst, vdst{7-0}, 0);
319}
320
321class VOP3OpSel_gfx9 <bits<10> op, VOPProfile P> : VOP3e_vi <op, P> {
322  let Inst{11} = !if(P.HasSrc0, src0_modifiers{2}, 0);
323  let Inst{12} = !if(P.HasSrc1, src1_modifiers{2}, 0);
324  let Inst{13} = !if(P.HasSrc2, src2_modifiers{2}, 0);
325  let Inst{14} = !if(P.HasDst,  src0_modifiers{3}, 0);
326}
327
328// Special case for v_permlane16_swap_b32/v_permlane32_swap_b32
329// op_sel[0]/op_sel[1] are treated as bound_ctrl and fi dpp operands.
330class VOP3OpSelIsDPP_gfx9 <bits<10> op, VOPProfile P> : VOP3e_vi <op, P> {
331  bits<1> fi;
332  bits<1> bound_ctrl;
333
334  // OPSEL[0] specifies FI
335  let Inst{11} = fi;
336  // OPSEL[1] specifies BOUND_CTRL
337  let Inst{12} = bound_ctrl;
338}
339
340class VOP3OpSel_gfx10<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
341  let Inst{11} = !if(p.HasSrc0, src0_modifiers{2}, 0);
342  let Inst{12} = !if(p.HasSrc1, src1_modifiers{2}, 0);
343  let Inst{13} = !if(p.HasSrc2, src2_modifiers{2}, 0);
344  let Inst{14} = !if(p.HasDst,  src0_modifiers{3}, 0);
345}
346
347class VOP3OpSel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3OpSel_gfx10<op, p>;
348
349class VOP3FP8OpSel_src_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
350  bits<2> byte_sel;
351  let Inst{11-12} = byte_sel; // NB: bit order is intentionally reversed!
352  let Inst{14-13} = 0;  // op_sel2/3
353}
354
355 class VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
356   bits<2> byte_sel;
357
358   let Inst{11} = 0; // op_sel0
359   let Inst{12} = 0; // op_sel1
360   let Inst{14-13} = byte_sel;  // op_sel2/3
361 }
362
363class VOP3DotOpSel_gfx11_gfx12<bits<10> op, VOPProfile p> :
364    VOP3e_t16_gfx11_gfx12<op, p>{
365  let Inst{11} = ?;
366  let Inst{12} = ?;
367  let Inst{13} = !if(p.HasSrc2Mods, src2_modifiers{2}, 0);
368  let Inst{14} = !if(!and(p.HasDst, p.HasSrc0Mods),  src0_modifiers{3}, 0);
369}
370
371// NB: For V_INTERP* opcodes, src0 is encoded as src1 and vice versa
372class VOP3Interp_vi <bits<10> op, VOPProfile P> : VOP3e_vi <op, P> {
373  bits<2> attrchan;
374  bits<6> attr;
375  bits<1> high;
376
377  let Inst{8}     = 0; // No modifiers for src0
378  let Inst{61}    = 0;
379
380  let Inst{9}     = !if(P.HasSrc0Mods, src0_modifiers{1}, 0);
381  let Inst{62}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0);
382
383  let Inst{37-32} = attr;
384  let Inst{39-38} = attrchan;
385  let Inst{40}    = !if(P.HasHigh, high, 0);
386
387  let Inst{49-41} = src0;
388}
389
390class VOP3Interp_gfx10<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
391  bits<6> attr;
392  bits<2> attrchan;
393  bits<1> high;
394
395  let Inst{8}     = 0;
396  let Inst{9}     = !if(p.HasSrc0Mods, src0_modifiers{1}, 0);
397  let Inst{37-32} = attr;
398  let Inst{39-38} = attrchan;
399  let Inst{40}    = !if(p.HasHigh, high, 0);
400  let Inst{49-41} = src0;
401  let Inst{61}    = 0;
402  let Inst{62}    = !if(p.HasSrc0Mods, src0_modifiers{0}, 0);
403}
404
405class VOP3Interp_gfx11<bits<10> op, VOPProfile p> : VOP3Interp_gfx10<op, p>;
406
407class VOP3be <VOPProfile P> : Enc64 {
408  bits<8> vdst;
409  bits<2> src0_modifiers;
410  bits<9> src0;
411  bits<2> src1_modifiers;
412  bits<9> src1;
413  bits<2> src2_modifiers;
414  bits<9> src2;
415  bits<7> sdst;
416  bits<2> omod;
417
418  let Inst{7-0}   = vdst;
419  let Inst{14-8}  = sdst;
420  let Inst{31-26} = 0x34; //encoding
421  let Inst{40-32} = !if(P.HasSrc0, src0, 0);
422  let Inst{49-41} = !if(P.HasSrc1, src1, 0);
423  let Inst{58-50} = !if(P.HasSrc2, src2, 0);
424  let Inst{60-59} = !if(P.HasOMod, omod, 0);
425  let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0);
426  let Inst{62}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0);
427  let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
428}
429
430class VOP3Pe <bits<7> op, VOPProfile P> : Enc64 {
431  bits<8> vdst;
432  bits<4> src0_modifiers;
433  bits<9> src0;
434  bits<4> src1_modifiers;
435  bits<9> src1;
436  bits<4> src2_modifiers;
437  bits<9> src2;
438  bits<1> clamp;
439  bits<2> index_key_8bit;
440  bits<1> index_key_16bit;
441
442  let Inst{7-0} = !if(P.HasDst, vdst, 0);
443  let Inst{8} = !if(P.HasSrc0Mods, src0_modifiers{1}, 0); // neg_hi src0
444  let Inst{9} = !if(P.HasSrc1Mods, src1_modifiers{1}, 0); // neg_hi src1
445  let Inst{10} = !if(P.HasSrc2Mods, src2_modifiers{1}, 0); // neg_hi src2
446
447  let Inst{11} = !if(!and(P.HasSrc0, P.HasOpSel), src0_modifiers{2}, 0); // op_sel(0)
448  let Inst{12} = !if(!and(P.HasSrc1, P.HasOpSel), src1_modifiers{2}, 0); // op_sel(1)
449  let Inst{13} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{2}, 0); // op_sel(2)
450
451  let Inst{14} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(2)
452
453  let Inst{15} = !if(P.HasClamp, clamp{0}, 0);
454
455  let Inst{22-16} = op;
456  let Inst{31-23} = 0x1a7; //encoding
457  let Inst{40-32} = !if(P.HasSrc0, src0, 0);
458  let Inst{49-41} = !if(P.HasSrc1, src1, 0);
459  let Inst{58-50} = !if(P.HasSrc2, src2, 0);
460  let Inst{59}    = !if(!and(P.HasSrc0, P.HasOpSel), src0_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(0)
461  let Inst{60}    = !if(!and(P.HasSrc1, P.HasOpSel), src1_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(1)
462  let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0); // neg (lo)
463  let Inst{62}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0); // neg (lo)
464  let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0); // neg (lo)
465}
466
467class VOP3Pe_MAI_Base {
468  bits<8> vdst;
469  bits<10> src0;
470  bits<10> src1;
471  bits<9> src2;
472  bits<3> blgp;
473  bits<3> cbsz;
474  bits<4> abid;
475}
476
477class VOP3Pe_MAI <bits<7> op, VOPProfile P, bit acc_cd = 0> : Enc64, VOP3Pe_MAI_Base {
478  let Inst{7-0} = vdst;
479
480  let Inst{10-8}  = !if(P.HasSrc1, cbsz, 0);
481  let Inst{14-11} = !if(P.HasAbid, abid, 0);
482
483  let Inst{15} = acc_cd;
484
485  let Inst{22-16} = op;
486  let Inst{31-23} = 0x1a7; //encoding
487  let Inst{40-32} = !if(P.HasSrc0, src0{8-0}, 0);
488  let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
489  let Inst{58-50} = !if(P.HasSrc2, src2, 0);
490
491  let Inst{59}    = !if(P.HasSrc0, src0{9}, 0); // acc(0)
492  let Inst{60}    = !if(P.HasSrc1, src1{9}, 0); // acc(1)
493
494  let Inst{63-61} = !if(P.HasSrc1, blgp, 0);
495}
496
497class VOP3Pe_SMFMAC <bits<7> op> : Enc64 {
498  bits<10> vdst; // VGPR or AGPR, but not SGPR. vdst{8} is not encoded in the instruction.
499  bits<10> src0;
500  bits<10> src1;
501  bits<9> idx;
502  bits<3> blgp;
503  bits<3> cbsz;
504  bits<4> abid;
505
506  let blgp = 0;
507
508  let Inst{7-0} = vdst{7-0};
509
510  let Inst{10-8}  = cbsz;
511  let Inst{14-11} = abid;
512
513  let Inst{15} = vdst{9}; // acc(vdst)
514
515  let Inst{22-16} = op;
516  let Inst{31-23} = 0x1a7; // encoding
517  let Inst{40-32} = src0{8-0};
518  let Inst{49-41} = src1{8-0};
519  let Inst{58-50} = idx;
520
521  let Inst{59}    = src0{9}; // acc(0)
522  let Inst{60}    = src1{9}; // acc(1)
523
524  let Inst{63-61} = blgp;
525}
526
527class VOP3PXe <bits<7> op, VOPProfile MFMAPfl, bit acc_cd = 0> : Enc128, VOP3Pe_MAI_Base {
528  bits<9> scale_src0;
529  bits<9> scale_src1;
530
531  bits<2> scale_src0_opsel;
532  bits<2> scale_src1_opsel;
533
534  // Inst{7-0} = unused
535  // Inst{10-8} = neg_hi;
536  // Inst{13-11} = op_sel
537  let Inst{11} = scale_src0_opsel{0};
538  let Inst{12} = scale_src1_opsel{0};
539  // Inst{13} = unused op_sel
540  // Inst{14} = unused op_sel_hi2
541
542  let Inst{31-16} = 0b1101001110101100;
543  let Inst{40-32} = scale_src0;
544  let Inst{49-41} = scale_src1;
545  // Inst{50-58} = unused
546  // Inst{60-59} = op_sel_hi;
547  let Inst{59} = scale_src0_opsel{1};
548  let Inst{60} = scale_src1_opsel{1};
549  // Inst{63-61} = neg;
550
551  // The high half of the encoding is the unscaled mfma op.
552  //
553  // FIXME: Defining the encoding in terms of the base instruction
554  // seems to not work, results in all 0 encoding, so replicate all
555  // the fields from VOP3Pe_MAI, shifted up by 64
556  //
557  // defvar Hi = VOP3Pe_MAI<op, MFMAPfl, acc_cd>;
558  // let Inst{127-64} = Hi.Inst;
559
560  let Inst{71-64} = vdst;
561  let Inst{74-72} = !if(MFMAPfl.HasSrc1, cbsz, 0);
562
563  // abid must be 1 to use a scale.
564  let Inst{78-75} = 0b0001; // abid
565
566  let Inst{79} = acc_cd;
567
568  let Inst{86-80} = op;
569  let Inst{95-87} = 0x1a7; //encoding
570  let Inst{104-96} = !if(MFMAPfl.HasSrc0, src0{8-0}, 0);
571  let Inst{113-105} = !if(MFMAPfl.HasSrc1, src1{8-0}, 0);
572  let Inst{122-114} = !if(MFMAPfl.HasSrc2, src2, 0);
573
574  let Inst{123}    = !if(MFMAPfl.HasSrc0, src0{9}, 0); // acc(0)
575  let Inst{124}    = !if(MFMAPfl.HasSrc1, src1{9}, 0); // acc(1)
576
577  let Inst{127-125} = !if(MFMAPfl.HasSrc1, blgp, 0);
578}
579
580class VOP3Pe_gfx10 <bits<7> op, VOPProfile P> : VOP3Pe<op, P> {
581  let Inst{31-23} = 0x198; //encoding
582}
583
584class VOP3Pe_gfx11_gfx12<bits<7> op, VOPProfile P> : VOP3Pe_gfx10<op, P>;
585
586class VOP3be_gfx6_gfx7<bits<9> op, VOPProfile p> : VOP3be<p> {
587  let Inst{25-17} = op;
588}
589
590class VOP3be_gfx10<bits<10> op, VOPProfile p> : VOP3be<p> {
591  bits<1> clamp;
592  let Inst{15}    = !if(p.HasClamp, clamp{0}, 0);
593  let Inst{25-16} = op;
594  let Inst{31-26} = 0x35;
595}
596
597class VOP3be_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3be_gfx10<op, p>;
598
599class VOP3be_vi <bits<10> op, VOPProfile P> : VOP3be<P> {
600  bits<1> clamp;
601  let Inst{25-16} = op;
602  let Inst{15}    = !if(P.HasClamp, clamp{0}, 0);
603}
604
605def SDWA {
606  // sdwa_sel
607  int BYTE_0 = 0;
608  int BYTE_1 = 1;
609  int BYTE_2 = 2;
610  int BYTE_3 = 3;
611  int WORD_0 = 4;
612  int WORD_1 = 5;
613  int DWORD = 6;
614
615  // dst_unused
616  int UNUSED_PAD = 0;
617  int UNUSED_SEXT = 1;
618  int UNUSED_PRESERVE = 2;
619}
620
621class VOP_SDWAe<VOPProfile P> : Enc64 {
622  bits<8> src0;
623  bits<3> src0_sel;
624  bits<2> src0_modifiers; // float: {abs,neg}, int {sext}
625  bits<3> src1_sel;
626  bits<2> src1_modifiers;
627  bits<3> dst_sel;
628  bits<2> dst_unused;
629  bits<1> clamp;
630
631  let Inst{39-32} = !if(P.HasSrc0, src0{7-0}, 0);
632  let Inst{42-40} = !if(P.EmitDstSel, dst_sel{2-0}, ?);
633  let Inst{44-43} = !if(P.EmitDstSel, dst_unused{1-0}, ?);
634  let Inst{45}    = !if(P.HasSDWAClamp, clamp{0}, 0);
635  let Inst{50-48} = !if(P.HasSrc0, src0_sel{2-0}, 0);
636  let Inst{51}    = !if(P.HasSrc0IntMods, src0_modifiers{0}, 0);
637  let Inst{53-52} = !if(P.HasSrc0FloatMods, src0_modifiers{1-0}, 0);
638  let Inst{58-56} = !if(P.HasSrc1, src1_sel{2-0}, 0);
639  let Inst{59}    = !if(P.HasSrc1IntMods, src1_modifiers{0}, 0);
640  let Inst{61-60} = !if(P.HasSrc1FloatMods, src1_modifiers{1-0}, 0);
641}
642
643// GFX9 adds two features to SDWA:
644// 1.	Add 3 fields to the SDWA microcode word: S0, S1 and OMOD.
645//    a. S0 and S1 indicate that source 0 and 1 respectively are SGPRs rather
646//       than VGPRs (at most 1 can be an SGPR);
647//    b. OMOD is the standard output modifier (result *2, *4, /2)
648// 2.	Add a new version of the SDWA microcode word for VOPC: SDWAB. This
649//    replaces OMOD and the dest fields with SD and SDST (SGPR destination)
650//    field.
651//    a. When SD=1, the SDST is used as the destination for the compare result;
652//    b. When SD=0, VCC is used.
653//
654// In GFX9, V_MAC_F16, V_MAC_F32 opcodes cannot be used with SDWA
655
656// gfx9 SDWA basic encoding
657class VOP_SDWA9e<VOPProfile P> : Enc64 {
658  bits<9> src0; // {src0_sgpr{0}, src0{7-0}}
659  bits<3> src0_sel;
660  bits<2> src0_modifiers; // float: {abs,neg}, int {sext}
661  bits<3> src1_sel;
662  bits<2> src1_modifiers;
663  bits<1> src1_sgpr;
664
665  let Inst{39-32} = !if(P.HasSrc0, src0{7-0}, 0);
666  let Inst{50-48} = !if(P.HasSrc0, src0_sel{2-0}, 0);
667  let Inst{51}    = !if(P.HasSrc0IntMods, src0_modifiers{0}, 0);
668  let Inst{53-52} = !if(P.HasSrc0FloatMods, src0_modifiers{1-0}, 0);
669  let Inst{55}    = !if(P.HasSrc0, src0{8}, 0);
670  let Inst{58-56} = !if(P.HasSrc1, src1_sel{2-0}, 0);
671  let Inst{59}    = !if(P.HasSrc1IntMods, src1_modifiers{0}, 0);
672  let Inst{61-60} = !if(P.HasSrc1FloatMods, src1_modifiers{1-0}, 0);
673  let Inst{63}    = 0; // src1_sgpr - should be specified in subclass
674}
675
676// gfx9 SDWA-A
677class VOP_SDWA9Ae<VOPProfile P> : VOP_SDWA9e<P> {
678  bits<3> dst_sel;
679  bits<2> dst_unused;
680  bits<1> clamp;
681  bits<2> omod;
682
683  let Inst{42-40} = !if(P.EmitDstSel, dst_sel{2-0}, ?);
684  let Inst{44-43} = !if(P.EmitDstSel, dst_unused{1-0}, ?);
685  let Inst{45}    = !if(P.HasSDWAClamp, clamp{0}, 0);
686  let Inst{47-46} = !if(P.HasSDWAOMod, omod{1-0}, 0);
687}
688
689// gfx9 SDWA-B
690class VOP_SDWA9Be<VOPProfile P> : VOP_SDWA9e<P> {
691  bits<8> sdst; // {vcc_sdst{0}, sdst{6-0}}
692
693  let Inst{46-40} = !if(P.EmitDst, sdst{6-0}, ?);
694  let Inst{47} = !if(P.EmitDst, sdst{7}, 0);
695}
696
697class VOP_SDWA_Pseudo <string opName, VOPProfile P, list<dag> pattern=[]> :
698  InstSI <P.OutsSDWA, P.InsSDWA, "", pattern>,
699  VOP <opName>,
700  SIMCInstr <opName#"_sdwa", SIEncodingFamily.NONE> {
701
702  let isPseudo = 1;
703  let isCodeGenOnly = 1;
704  let UseNamedOperandTable = 1;
705
706  string Mnemonic = opName;
707  string AsmOperands = P.AsmSDWA;
708  string AsmOperands9 = P.AsmSDWA9;
709
710  let Size = 8;
711  let mayLoad = 0;
712  let mayStore = 0;
713  let hasSideEffects = 0;
714
715  let VALU = 1;
716  let SDWA = 1;
717
718  let ReadsModeReg = !or(P.DstVT.isFP, P.Src0VT.isFP);
719
720  let mayRaiseFPException = ReadsModeReg;
721  let Uses = !if(ReadsModeReg, [MODE, EXEC], [EXEC]);
722
723  let SubtargetPredicate = HasSDWA;
724  let AsmVariantName = !if(P.HasExtSDWA, AMDGPUAsmVariants.SDWA,
725                                         AMDGPUAsmVariants.Disable);
726  let DecoderNamespace = "GFX8";
727
728  VOPProfile Pfl = P;
729}
730
731class VOP_SDWA8_Real <VOP_SDWA_Pseudo ps> :
732  InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>,
733  SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SDWA> {
734
735  let VALU = 1;
736  let SDWA = 1;
737  let isPseudo = 0;
738  let isCodeGenOnly = 0;
739
740  let Defs = ps.Defs;
741  let Uses = ps.Uses;
742  let hasSideEffects = ps.hasSideEffects;
743
744  let Constraints     = ps.Constraints;
745  let DisableEncoding = ps.DisableEncoding;
746
747  // Copy relevant pseudo op flags
748  let SubtargetPredicate   = ps.SubtargetPredicate;
749  let AssemblerPredicate   = HasSDWA8;
750  let AsmMatchConverter    = ps.AsmMatchConverter;
751  let AsmVariantName       = ps.AsmVariantName;
752  let UseNamedOperandTable = ps.UseNamedOperandTable;
753  let DecoderNamespace     = ps.DecoderNamespace;
754  let Constraints          = ps.Constraints;
755  let DisableEncoding      = ps.DisableEncoding;
756  let TSFlags              = ps.TSFlags;
757  let Uses                 = ps.Uses;
758  let Defs                 = ps.Defs;
759  let SchedRW              = ps.SchedRW;
760  let mayLoad              = ps.mayLoad;
761  let mayStore             = ps.mayStore;
762  let TRANS                = ps.TRANS;
763  let isConvergent         = ps.isConvergent;
764}
765
766class Base_VOP_SDWA9_Real <VOP_SDWA_Pseudo ps> :
767  InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands9, []> {
768
769  let VALU = 1;
770  let SDWA = 1;
771  let isPseudo = 0;
772  let isCodeGenOnly = 0;
773
774  let Defs = ps.Defs;
775  let Uses = ps.Uses;
776  let hasSideEffects = ps.hasSideEffects;
777
778  let Constraints     = ps.Constraints;
779  let DisableEncoding = ps.DisableEncoding;
780
781  let SubtargetPredicate = ps.SubtargetPredicate;
782  let AssemblerPredicate = HasSDWA9;
783  let OtherPredicates    = ps.OtherPredicates;
784  let AsmVariantName = !if(ps.Pfl.HasExtSDWA9, AMDGPUAsmVariants.SDWA9,
785                                               AMDGPUAsmVariants.Disable);
786  let DecoderNamespace = "GFX9";
787
788  // Copy relevant pseudo op flags
789  let AsmMatchConverter    = ps.AsmMatchConverter;
790  let UseNamedOperandTable = ps.UseNamedOperandTable;
791  let Constraints          = ps.Constraints;
792  let DisableEncoding      = ps.DisableEncoding;
793  let TSFlags              = ps.TSFlags;
794  let Uses                 = ps.Uses;
795  let Defs                 = ps.Defs;
796  let SchedRW              = ps.SchedRW;
797  let mayLoad              = ps.mayLoad;
798  let mayStore             = ps.mayStore;
799  let TRANS                = ps.TRANS;
800  let isConvergent         = ps.isConvergent;
801}
802
803class VOP_SDWA9_Real <VOP_SDWA_Pseudo ps> :
804  Base_VOP_SDWA9_Real <ps >,
805  SIMCInstr <ps.PseudoInstr, SIEncodingFamily.SDWA9>;
806
807class Base_VOP_SDWA10_Real<VOP_SDWA_Pseudo ps> : Base_VOP_SDWA9_Real<ps> {
808  let SubtargetPredicate = ps.SubtargetPredicate;
809  let AssemblerPredicate = HasSDWA10;
810  let DecoderNamespace = "GFX10";
811}
812
813class VOP_SDWA10_Real<VOP_SDWA_Pseudo ps> :
814  Base_VOP_SDWA10_Real<ps>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SDWA10>;
815
816class VOP_DPPe<VOPProfile P, bit IsDPP16=0> : Enc64 {
817  bits<2> src0_modifiers;
818  bits<8> src0;
819  bits<2> src1_modifiers;
820  bits<9> dpp_ctrl;
821  bits<1> bound_ctrl;
822  bits<4> bank_mask;
823  bits<4> row_mask;
824  bit     fi;
825
826  let Inst{39-32} = !if(P.HasSrc0, src0{7-0}, 0);
827  let Inst{48-40} = dpp_ctrl;
828  let Inst{50}    = !if(IsDPP16, fi, ?);
829  let Inst{51}    = bound_ctrl;
830  let Inst{52}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0); // src0_neg
831  let Inst{53}    = !if(P.HasSrc0Mods, src0_modifiers{1}, 0); // src0_abs
832  let Inst{54}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0); // src1_neg
833  let Inst{55}    = !if(P.HasSrc1Mods, src1_modifiers{1}, 0); // src1_abs
834  let Inst{59-56} = bank_mask;
835  let Inst{63-60} = row_mask;
836}
837
838class VOP3_DPPe_Fields_Base {
839  bits<9> dpp_ctrl;
840  bits<1> bound_ctrl;
841  bits<4> bank_mask;
842  bits<4> row_mask;
843  bit     fi;
844}
845class VOP3_DPPe_Fields : VOP3_DPPe_Fields_Base {
846  bits<8> src0;
847}
848
849class VOP3_DPPe_Fields_t16 : VOP3_DPPe_Fields_Base {
850  bits<11> src0;
851}
852
853// Common refers to common between DPP and DPP8
854// Base refers to a shared base between T16 and regular instructions
855class VOP3_DPPe_Common_Base<bits<10> op, VOPProfile P> : Enc96 {
856  bits<4> src0_modifiers;
857  bits<3> src1_modifiers;
858  bits<3> src2_modifiers;
859  bits<1> clamp;
860  bits<2> omod;
861  bits<2> byte_sel;
862
863  let Inst{8}     = !if(P.HasSrc0Mods, src0_modifiers{1}, 0);
864  let Inst{9}     = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
865  let Inst{10}    = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
866  // 16-bit select fields which can be interpreted as OpSel or hi/lo suffix
867  let Inst{11} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{2}, 0),
868                                 !if(P.IsFP8SrcByteSel, byte_sel{1}, ?));
869  let Inst{12} = !if(P.HasOpSel, !if(P.HasSrc1Mods, src1_modifiers{2}, 0),
870                                 !if(P.IsFP8SrcByteSel, byte_sel{0}, ?));
871  let Inst{13} = !if(P.HasOpSel, !if(P.HasSrc2Mods, src2_modifiers{2}, 0),
872                                 !if(P.IsFP8DstByteSel, byte_sel{0}, ?));
873  let Inst{14} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{3}, 0),
874                                 !if(P.IsFP8DstByteSel, byte_sel{1}, ?));
875  let Inst{15}    = !if(P.HasClamp, clamp, 0);
876  let Inst{25-16} = op;
877  let Inst{31-26} = 0x35;
878
879  let Inst{60-59} = !if(P.HasOMod, omod, 0);
880  let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0);
881  let Inst{62}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0);
882  let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0);
883}
884
885class VOP3_DPPe_Common<bits<10> op, VOPProfile P> : VOP3_DPPe_Common_Base<op, P> {
886  bits<8> vdst;
887  bits<9> src1;
888  bits<9> src2;
889
890  let Inst{7-0}   = !if(P.EmitDst, vdst{7-0}, 0);
891  let Inst{49-41} = !if(P.HasSrc1, src1, 0);
892  let Inst{58-50} = !if(P.HasSrc2, src2, 0);
893}
894
895class VOP3_DPPe_Common_t16<bits<10> op, VOPProfile P> : VOP3_DPPe_Common_Base<op, P> {
896  bits<11> vdst;
897  bits<11> src1;
898  bits<11> src2;
899
900  let Inst{7-0}   = !if(P.EmitDst, vdst{7-0}, 0);
901  let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
902  let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
903}
904
905class VOP3P_DPPe_Common_Base<bits<7> op, VOPProfile P> : Enc96 {
906  bits<4> src0_modifiers;
907  bits<4> src1_modifiers;
908  bits<4> src2_modifiers;
909  bits<1> clamp;
910
911  let Inst{8} = !if(P.HasSrc0Mods, src0_modifiers{1}, 0); // neg_hi src0
912  let Inst{9} = !if(P.HasSrc1Mods, src1_modifiers{1}, 0); // neg_hi src1
913  let Inst{10} = !if(P.HasSrc2Mods, src2_modifiers{1}, 0); // neg_hi src2
914  // OPSEL must be set such that the low result only uses low inputs, and the high result only uses high inputs.
915  let Inst{11} = !if(!and(P.HasSrc0, P.HasOpSel), src0_modifiers{2}, 0); // op_sel(0)
916  let Inst{12} = !if(!and(P.HasSrc1, P.HasOpSel), src1_modifiers{2}, 0); // op_sel(1)
917  let Inst{13} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{2}, 0); // op_sel(2)
918  let Inst{14} = !if(!and(P.HasSrc2, P.HasOpSel), src2_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(2)
919  let Inst{15} = !if(P.HasClamp, clamp{0}, 0);
920  let Inst{22-16} = op;
921  let Inst{31-23} = 0x198; // encoding
922  let Inst{59}    = !if(!and(P.HasSrc0, P.HasOpSel), src0_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(0)
923  let Inst{60}    = !if(!and(P.HasSrc1, P.HasOpSel), src1_modifiers{3}, !if(P.IsDOT, 1, ?)); // op_sel_hi(1)
924  let Inst{61}    = !if(P.HasSrc0Mods, src0_modifiers{0}, 0); // neg (lo)
925  let Inst{62}    = !if(P.HasSrc1Mods, src1_modifiers{0}, 0); // neg (lo)
926  let Inst{63}    = !if(P.HasSrc2Mods, src2_modifiers{0}, 0); // neg (lo)
927}
928
929class VOP3P_DPPe_Common<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
930  bits<8> vdst;
931  bits<9> src1;
932  bits<9> src2;
933
934  let Inst{7-0} = vdst;
935  let Inst{49-41} = !if(P.HasSrc1, src1, 0);
936  let Inst{58-50} = !if(P.HasSrc2, src2, 0);
937}
938
939class VOP3P_DPPe_Common_t16<bits<7> op, VOPProfile P> : VOP3P_DPPe_Common_Base<op, P> {
940  bits<11> vdst;
941  bits<11> src1;
942  bits<11> src2;
943
944  let Inst{7-0} = vdst{7-0};
945  let Inst{49-41} = !if(P.HasSrc1, src1{8-0}, 0);
946  let Inst{58-50} = !if(P.HasSrc2, src2{8-0}, 0);
947}
948
949class VOP_DPP_Pseudo <string OpName, VOPProfile P, list<dag> pattern=[],
950  dag Ins = P.InsDPP, string asmOps = P.AsmDPP> :
951  VOP_Pseudo<OpName, "_dpp", P, P.OutsDPP, Ins, asmOps, pattern> {
952
953  let mayLoad = 0;
954  let mayStore = 0;
955  let hasSideEffects = 0;
956
957  let VALU = 1;
958  let DPP = 1;
959  let Size = 8;
960  let IsPacked = P.IsPacked;
961
962  let ReadsModeReg = !or(P.DstVT.isFP, P.Src0VT.isFP);
963
964  let mayRaiseFPException = ReadsModeReg;
965  let Uses = !if(ReadsModeReg, [MODE, EXEC], [EXEC]);
966  let isConvergent = 1;
967
968  string AsmOperands = asmOps;
969
970  let AsmMatchConverter = !if(P.HasModifiers, "cvtDPP", "");
971  let AssemblerPredicate = !if(P.HasExt64BitDPP, HasDPALU_DPP, HasDPP);
972  let AsmVariantName = !if(P.HasExtDPP, AMDGPUAsmVariants.DPP,
973                                        AMDGPUAsmVariants.Disable);
974  let Constraints = !if(P.NumSrcArgs, P.TieRegDPP # " = $vdst", "");
975  let DisableEncoding = !if(P.NumSrcArgs, P.TieRegDPP, "");
976  let DecoderNamespace = "GFX8";
977}
978
979class VOP3_DPP_Pseudo <string OpName, VOPProfile P> :
980  VOP_DPP_Pseudo <OpName, P, [], P.InsVOP3DPP, P.AsmVOP3DPP> {
981  let PseudoInstr = OpName#"_e64"#"_dpp";
982  let OutOperandList = P.OutsVOP3DPP;
983  let Size = 12;
984  let VOP3 = 1;
985  let AsmMatchConverter = "cvtVOP3DPP";
986  let AsmVariantName = !if(P.HasExtVOP3DPP, AMDGPUAsmVariants.VOP3_DPP,
987                                            AMDGPUAsmVariants.Disable);
988}
989
990class VOP_DPP_Real <VOP_DPP_Pseudo ps, int EncodingFamily> :
991  InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []>,
992  SIMCInstr <ps.PseudoInstr, EncodingFamily> {
993
994  let VALU = 1;
995  let DPP = 1;
996  let isPseudo = 0;
997  let isCodeGenOnly = 0;
998
999  let Defs = ps.Defs;
1000  let Uses = ps.Uses;
1001  let hasSideEffects = ps.hasSideEffects;
1002
1003  let Constraints     = ps.Constraints;
1004  let DisableEncoding = ps.DisableEncoding;
1005
1006  // Copy relevant pseudo op flags
1007  let isConvergent         = ps.isConvergent;
1008  let SubtargetPredicate   = ps.SubtargetPredicate;
1009  let True16Predicate      = ps.True16Predicate;
1010  let AssemblerPredicate   = ps.AssemblerPredicate;
1011  let OtherPredicates      = ps.OtherPredicates;
1012  let AsmMatchConverter    = ps.AsmMatchConverter;
1013  let AsmVariantName       = ps.AsmVariantName;
1014  let UseNamedOperandTable = ps.UseNamedOperandTable;
1015  let DecoderNamespace     = ps.DecoderNamespace;
1016  let Constraints          = ps.Constraints;
1017  let DisableEncoding      = ps.DisableEncoding;
1018  let TSFlags              = ps.TSFlags;
1019  let Uses                 = ps.Uses;
1020  let Defs                 = ps.Defs;
1021  let SchedRW              = ps.SchedRW;
1022  let mayLoad              = ps.mayLoad;
1023  let mayStore             = ps.mayStore;
1024  let TRANS                = ps.TRANS;
1025  let isConvergent         = ps.isConvergent;
1026}
1027
1028class VOP_DPP_Base <string OpName, VOPProfile P,
1029               dag InsDPP,
1030               string AsmDPP > :
1031  InstSI <P.OutsDPP, InsDPP, OpName#AsmDPP, []> {
1032
1033  let mayLoad = 0;
1034  let mayStore = 0;
1035  let hasSideEffects = 0;
1036  let UseNamedOperandTable = 1;
1037
1038  let VALU = 1;
1039  let DPP = 1;
1040  let Size = 8;
1041
1042  let AsmMatchConverter = !if(P.HasModifiers, "cvtDPP", "");
1043  let AssemblerPredicate = !if(P.HasExt64BitDPP, HasDPALU_DPP, HasDPP);
1044  let AsmVariantName = !if(P.HasExtDPP, AMDGPUAsmVariants.DPP,
1045                                        AMDGPUAsmVariants.Disable);
1046  let Constraints = !if(P.NumSrcArgs, P.TieRegDPP # " = $vdst", "");
1047  let DisableEncoding = !if(P.NumSrcArgs, P.TieRegDPP, "");
1048  let DecoderNamespace = "GFX8";
1049}
1050
1051class VOP_DPP <string OpName, VOPProfile P, bit IsDPP16,
1052               dag InsDPP = !if(IsDPP16, P.InsDPP16, P.InsDPP),
1053               string AsmDPP = !if(IsDPP16, P.AsmDPP16, P.AsmDPP)> :
1054  VOP_DPP_Base<OpName, P, InsDPP, AsmDPP>, VOP_DPPe<P, IsDPP16>;
1055
1056class VOP3_DPP_Base <string OpName, VOPProfile P, bit IsDPP16,
1057               dag InsDPP = !if(IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
1058               string AsmDPP = !if(IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)> :
1059  VOP_DPP_Base<OpName, P, InsDPP, AsmDPP> {
1060  let OutOperandList = P.OutsVOP3DPP;
1061  let AsmMatchConverter = "cvtVOP3DPP";
1062  let VOP3 = 1;
1063  let AsmVariantName = !if(P.HasExtVOP3DPP, AMDGPUAsmVariants.VOP3_DPP,
1064                                            AMDGPUAsmVariants.Disable);
1065  let Size = 12;
1066}
1067
1068class VOP3_DPP_Enc <bits<10> op, VOPProfile P, bit IsDPP16> :
1069  VOP3_DPPe_Common<op, P>,
1070  VOP3_DPPe_Fields {
1071
1072  let Inst{40-32} = 0xfa;
1073  let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
1074  let Inst{80-72} = dpp_ctrl;
1075  let Inst{82}    = !if(IsDPP16, fi, ?);
1076  let Inst{83}    = bound_ctrl;
1077
1078  // Inst{87-84} ignored by hw
1079  let Inst{91-88} = bank_mask;
1080  let Inst{95-92} = row_mask;
1081}
1082
1083class VOP3_DPP <bits<10> op, string OpName, VOPProfile P, bit IsDPP16,
1084               dag InsDPP = !if(IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
1085               string AsmDPP = !if(IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)> :
1086  VOP3_DPP_Base<OpName, P, IsDPP16, InsDPP, AsmDPP>, VOP3_DPP_Enc<op, P, IsDPP16>;
1087
1088class VOP3_DPP_Enc_t16<bits<10> op, VOPProfile P, bit IsDPP16 >
1089    : VOP3_DPPe_Common_t16<op, P>,
1090      VOP3_DPPe_Fields_t16 {
1091
1092  let Inst{40-32} = 0xfa;
1093  let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
1094  let Inst{80-72} = dpp_ctrl;
1095  let Inst{82}    = !if(IsDPP16, fi, ?);
1096  let Inst{83}    = bound_ctrl;
1097
1098  // Inst{87-84} ignored by hw
1099  let Inst{91-88} = bank_mask;
1100  let Inst{95-92} = row_mask;
1101}
1102
1103class VOP3_DPP_t16<bits<10> op, string OpName, VOPProfile P, bit IsDPP16,
1104                   dag InsDPP = !if (IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
1105                   string AsmDPP = !if (IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)>
1106    : VOP3_DPP_Base<OpName, P, IsDPP16, InsDPP, AsmDPP>,
1107      VOP3_DPP_Enc_t16<op, P, IsDPP16> {
1108}
1109
1110class VOP3P_DPP <bits<7> op, string OpName, VOPProfile P, bit IsDPP16,
1111               dag InsDPP = !if(IsDPP16, P.InsVOP3DPP16, P.InsVOP3DPP),
1112               string AsmDPP = !if(IsDPP16, P.AsmVOP3DPP16, P.AsmVOP3DPP)> :
1113  VOP3_DPP_Base<OpName, P, IsDPP16, InsDPP, AsmDPP>, VOP3P_DPPe_Common<op, P>,
1114  VOP3_DPPe_Fields {
1115
1116  let VOP3P = 1;
1117
1118  let Inst{40-32} = 0xfa;
1119  let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
1120  let Inst{80-72} = dpp_ctrl;
1121  let Inst{82}    = !if(IsDPP16, fi, ?);
1122  let Inst{83}    = bound_ctrl;
1123
1124  // Inst{87-84} ignored by hw
1125  let Inst{91-88} = bank_mask;
1126  let Inst{95-92} = row_mask;
1127}
1128
1129class VOP_DPP8e<VOPProfile P> : Enc64 {
1130  bits<8> src0;
1131  bits<24> dpp8;
1132  bits<9> fi;
1133
1134  let Inst{39-32} = !if(P.HasSrc0, src0{7-0}, 0);
1135  let Inst{63-40} = dpp8{23-0};
1136}
1137
1138class VOP3_DPP8e_Fields {
1139  bits<8> src0;
1140  bits<24> dpp8;
1141  bits<9> fi;
1142}
1143
1144class VOP3_DPP8e_Fields_t16 {
1145  bits<11> src0;
1146  bits<24> dpp8;
1147  bits<9> fi;
1148}
1149
1150class VOP_DPP8_Base<string OpName, VOPProfile P, dag InsDPP8 = P.InsDPP8, string AsmDPP8 = P.AsmDPP8> :
1151  InstSI<P.OutsDPP8, InsDPP8, OpName#AsmDPP8, []> {
1152
1153  let mayLoad = 0;
1154  let mayStore = 0;
1155  let hasSideEffects = 0;
1156  let UseNamedOperandTable = 1;
1157
1158  let VALU = 1;
1159  let DPP = 1;
1160  let Size = 8;
1161
1162  let AsmMatchConverter = "cvtDPP8";
1163  let AssemblerPredicate = HasDPP8;
1164  let AsmVariantName = AMDGPUAsmVariants.DPP;
1165  let Constraints = !if(P.NumSrcArgs, P.TieRegDPP # " = $vdst", "");
1166  let DisableEncoding = !if(P.NumSrcArgs, P.TieRegDPP, "");
1167}
1168
1169class VOP_DPP8<string OpName, VOPProfile P> :
1170  VOP_DPP8_Base<OpName, P>, VOP_DPP8e<P>;
1171
1172class VOP3_DPP8_Base<string OpName, VOPProfile P> :
1173  VOP_DPP8_Base<OpName, P, P.InsVOP3DPP8, P.AsmVOP3DPP8> {
1174  let OutOperandList = P.OutsVOP3DPP8;
1175  let AsmMatchConverter = "cvtVOP3DPP8";
1176  let AsmVariantName = !if(P.HasExtVOP3DPP, AMDGPUAsmVariants.VOP3_DPP,
1177                                            AMDGPUAsmVariants.Disable);
1178  let VOP3 = 1;
1179  let Size = 12;
1180}
1181
1182class VOP3_DPP8_Enc <bits<10> op, VOPProfile P> :
1183  VOP3_DPPe_Common<op, P>,
1184  VOP3_DPP8e_Fields {
1185  let Inst{40-32} = fi;
1186  let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
1187  let Inst{95-72} = dpp8{23-0};
1188}
1189
1190class VOP3_DPP8<bits<10> op, string OpName, VOPProfile P> :
1191  VOP3_DPP8_Base<OpName, P>, VOP3_DPP8_Enc<op, P>;
1192
1193class VOP3_DPP8_Enc_t16 <bits<10> op, VOPProfile P> :
1194  VOP3_DPPe_Common_t16<op, P>,
1195  VOP3_DPP8e_Fields_t16 {
1196  let Inst{40-32} = fi;
1197  let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
1198  let Inst{95-72} = dpp8{23-0};
1199}
1200
1201class VOP3_DPP8_t16<bits<10> op, string OpName, VOPProfile P> :
1202  VOP3_DPP8_Base<OpName, P>, VOP3_DPP8_Enc_t16<op, P>;
1203
1204class VOP3P_DPP8<bits<7> op, string OpName, VOPProfile P> :
1205  VOP3_DPP8_Base<OpName, P>, VOP3P_DPPe_Common<op, P>,
1206  VOP3_DPP8e_Fields {
1207
1208  let VOP3P = 1;
1209  let Inst{40-32} = fi;
1210  let Inst{71-64} = !if(P.HasSrc0, src0{7-0}, 0);
1211  let Inst{95-72} = dpp8{23-0};
1212}
1213
1214def DPP8Mode {
1215  int FI_0 = 0xE9;
1216  int FI_1 = 0xEA;
1217}
1218
1219class getNumNodeArgs<SDPatternOperator Op> {
1220  SDNode N = !cast<SDNode>(Op);
1221  SDTypeProfile TP = N.TypeProfile;
1222  int ret = TP.NumOperands;
1223}
1224
1225class getDivergentFrag<SDPatternOperator Op> {
1226  assert !or(!isa<SDNode>(Op), !isa<PatFrags>(Op)), "Expected SDNode or PatFrags";
1227
1228  int NumSrcArgs = !if(!isa<SDNode>(Op), getNumNodeArgs<Op>.ret,
1229    !size(!cast<PatFrags>(Op).Operands));
1230  PatFrag ret = PatFrag <
1231    !if(!eq(NumSrcArgs, 1),
1232             (ops node:$src0),
1233             !if(!eq(NumSrcArgs, 2),
1234               (ops node:$src0, node:$src1),
1235               (ops node:$src0, node:$src1, node:$src2))),
1236    !if(!eq(NumSrcArgs, 1),
1237             (Op $src0),
1238             !if(!eq(NumSrcArgs, 2),
1239               (Op $src0, $src1),
1240               (Op $src0, $src1, $src2))),
1241    [{ return N->isDivergent(); }]
1242  >;
1243}
1244
1245class VOPPatGen<SDPatternOperator Op, VOPProfile P> {
1246  PatFrag Operator = getDivergentFrag < Op >.ret;
1247
1248  dag Ins = !foreach(tmp, P.Ins32, !subst(ins, Operator,
1249                                         !subst(P.Src0RC32, P.Src0VT,
1250                                               !subst(P.Src1RC32, P.Src1VT, tmp))));
1251
1252  dag Outs = !foreach(tmp, P.Outs32, !subst(outs, set,
1253                                           !subst(P.DstRC, P.DstVT, tmp)));
1254
1255  list<dag> ret =  [!con(Outs, (set Ins))];
1256}
1257
1258class DivergentUnaryFrag<SDPatternOperator Op> : PatFrag <
1259  (ops node:$src0),
1260  (Op $src0),
1261  [{ return N->isDivergent(); }]> {
1262  // This check is unnecessary as it's captured by the result register
1263  // bank constraint.
1264  //
1265  // FIXME: Should add a way for the emitter to recognize this is a
1266  // trivially true predicate to eliminate the check.
1267  let GISelPredicateCode = [{return true;}];
1268}
1269
1270class VOPPatOrNull<SDPatternOperator Op, VOPProfile P> {
1271  list<dag> ret = !if(!ne(P.NeedPatGen,PatGenMode.NoPattern), VOPPatGen<Op, P>.ret, []);
1272}
1273
1274class DivergentFragOrOp<SDPatternOperator Op, VOPProfile P> {
1275  SDPatternOperator ret = !if(!eq(P.NeedPatGen,PatGenMode.Pattern),
1276   !if(!isa<SDNode>(Op), getDivergentFrag<Op>.ret, Op), Op);
1277}
1278
1279class getVSrcOp<ValueType vt> {
1280  RegisterOperand ret = !if(!eq(vt.Size, 32), VSrc_b32, VSrc_b16);
1281}
1282
1283// Class for binary integer operations with the clamp bit set for saturation
1284// TODO: Add sub with negated inline constant pattern.
1285class VOPBinOpClampPat<SDPatternOperator node, Instruction inst, ValueType vt> :
1286  GCNPat<(node vt:$src0, vt:$src1),
1287         (inst getVSrcOp<vt>.ret:$src0, getVSrcOp<vt>.ret:$src1,
1288               DSTCLAMP.ENABLE)
1289>;
1290
1291//===----------------------------------------------------------------------===//
1292// VOP3 Classes
1293//===----------------------------------------------------------------------===//
1294
1295class getVOP3ModPat<VOPProfile P, SDPatternOperator node> {
1296  dag src0 = !if(P.HasOMod,
1297    (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omod),
1298    (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers, i1:$clamp));
1299
1300  list<dag> ret3 = [(set P.DstVT:$vdst,
1301    (DivergentFragOrOp<node, P>.ret (P.Src0VT src0),
1302          (P.Src1VT (VOP3Mods P.Src1VT:$src1, i32:$src1_modifiers)),
1303          (P.Src2VT (VOP3Mods P.Src2VT:$src2, i32:$src2_modifiers))))];
1304
1305  list<dag> ret2 = [(set P.DstVT:$vdst,
1306    (DivergentFragOrOp<node, P>.ret (P.Src0VT src0),
1307          (P.Src1VT (VOP3Mods P.Src1VT:$src1, i32:$src1_modifiers))))];
1308
1309  list<dag> ret1 = [(set P.DstVT:$vdst,
1310    (DivergentFragOrOp<node, P>.ret (P.Src0VT src0)))];
1311
1312  list<dag> ret = !if(!eq(P.NumSrcArgs, 3), ret3,
1313                  !if(!eq(P.NumSrcArgs, 2), ret2,
1314                  ret1));
1315}
1316
1317class getVOP3PModPat<VOPProfile P, SDPatternOperator node, bit HasExplicitClamp,
1318                     bit IsDOT = 0,
1319                     ComplexPattern SrcPat = !if(IsDOT, VOP3PModsDOT, VOP3PMods)> {
1320  dag src0_dag = (P.Src0VT (SrcPat P.Src0VT:$src0, i32:$src0_modifiers));
1321  dag src1_dag = (P.Src1VT (SrcPat P.Src1VT:$src1, i32:$src1_modifiers));
1322  dag src2_dag = (P.Src2VT (SrcPat P.Src2VT:$src2, i32:$src2_modifiers));
1323  dag clamp_dag = (i1 timm:$clamp);
1324
1325  list<dag> ret3 = [(set P.DstVT:$vdst,
1326    !if(HasExplicitClamp,
1327        (DivergentFragOrOp<node, P>.ret src0_dag, src1_dag, src2_dag, clamp_dag),
1328        (DivergentFragOrOp<node, P>.ret src0_dag, src1_dag, src2_dag)))];
1329
1330  list<dag> ret2 = [(set P.DstVT:$vdst,
1331    !if(HasExplicitClamp,
1332        (DivergentFragOrOp<node, P>.ret src0_dag, src1_dag, clamp_dag),
1333        (DivergentFragOrOp<node, P>.ret src0_dag, src1_dag)))];
1334
1335  list<dag> ret1 = [(set P.DstVT:$vdst,
1336    !if(HasExplicitClamp,
1337        (DivergentFragOrOp<node, P>.ret src0_dag, clamp_dag),
1338        (DivergentFragOrOp<node, P>.ret src0_dag)))];
1339
1340  list<dag> ret = !if(!eq(P.NumSrcArgs, 3), ret3,
1341                  !if(!eq(P.NumSrcArgs, 2), ret2,
1342                  ret1));
1343}
1344
1345class getVOP3OpSelPat<VOPProfile P, SDPatternOperator node> {
1346  list<dag> ret3 = [(set P.DstVT:$vdst,
1347        (DivergentFragOrOp<node, P>.ret (P.Src0VT (VOP3OpSel P.Src0VT:$src0, i32:$src0_modifiers)),
1348          (P.Src1VT (VOP3OpSel P.Src1VT:$src1, i32:$src1_modifiers)),
1349          (P.Src2VT (VOP3OpSel P.Src2VT:$src2, i32:$src2_modifiers))))];
1350
1351  list<dag> ret2 = [(set P.DstVT:$vdst,
1352    (DivergentFragOrOp<node, P>.ret (P.Src0VT (VOP3OpSel P.Src0VT:$src0, i32:$src0_modifiers)),
1353                                    (P.Src1VT (VOP3OpSel P.Src1VT:$src1, i32:$src1_modifiers))))];
1354
1355  list<dag> ret1 = [(set P.DstVT:$vdst,
1356    (DivergentFragOrOp<node, P>.ret (P.Src0VT (VOP3OpSel P.Src0VT:$src0, i32:$src0_modifiers))))];
1357
1358  list<dag> ret = !if(!eq(P.NumSrcArgs, 3), ret3,
1359                  !if(!eq(P.NumSrcArgs, 2), ret2,
1360                  ret1));
1361}
1362
1363class getVOP3OpSelModPat<VOPProfile P, SDPatternOperator node> {
1364  list<dag> ret3 = [(set P.DstVT:$vdst,
1365    (DivergentFragOrOp<node, P>.ret (P.Src0VT !if(P.HasClamp, (VOP3OpSelMods P.Src0VT:$src0, i32:$src0_modifiers),
1366                                    (VOP3OpSelMods P.Src0VT:$src0, i32:$src0_modifiers))),
1367          (P.Src1VT (VOP3OpSelMods P.Src1VT:$src1, i32:$src1_modifiers)),
1368          (P.Src2VT (VOP3OpSelMods P.Src2VT:$src2, i32:$src2_modifiers))))];
1369
1370  list<dag> ret2 = [(set P.DstVT:$vdst,
1371    (DivergentFragOrOp<node, P>.ret !if(P.HasClamp, (P.Src0VT (VOP3OpSelMods P.Src0VT:$src0, i32:$src0_modifiers)),
1372                          (P.Src0VT (VOP3OpSelMods P.Src0VT:$src0, i32:$src0_modifiers))),
1373          (P.Src1VT (VOP3OpSelMods P.Src1VT:$src1, i32:$src1_modifiers))))];
1374
1375  list<dag> ret1 = [(set P.DstVT:$vdst,
1376    (DivergentFragOrOp<node, P>.ret (P.Src0VT (VOP3OpSelMods P.Src0VT:$src0, i32:$src0_modifiers))))];
1377
1378  list<dag> ret = !if(!eq(P.NumSrcArgs, 3), ret3,
1379                  !if(!eq(P.NumSrcArgs, 2), ret2,
1380                  ret1));
1381}
1382
1383class getVOP3FromVOP2Pat<VOPProfile P, SDPatternOperator node> {
1384  list<dag> ret = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1))];
1385}
1386// In VOP1, we can have clamp and omod even if !HasModifiers
1387class getVOP3Pat<VOPProfile P, SDPatternOperator node> {
1388  dag src0 =
1389    !if(P.HasOMod,
1390      !if(P.HasClamp,
1391          (VOP3Mods0 P.Src0VT:$src0, i1:$clamp, i32:$omod),
1392          (VOP3Mods0 P.Src0VT:$src0, i32:$omod)), // impossible?
1393      !if(P.HasClamp,
1394          (VOP3Mods0 P.Src0VT:$src0, i1:$clamp),
1395          (VOP3Mods0 P.Src0VT:$src0))
1396    );
1397  list<dag> ret3 = [(set P.DstVT:$vdst, (DivergentFragOrOp<node, P>.ret (P.Src0VT src0), P.Src1VT:$src1, P.Src2VT:$src2))];
1398
1399  list<dag> ret2 = [(set P.DstVT:$vdst, (DivergentFragOrOp<node, P>.ret (P.Src0VT src0), P.Src1VT:$src1))];
1400
1401  list<dag> ret1 = [(set P.DstVT:$vdst, (DivergentFragOrOp<node, P>.ret (P.Src0VT src0)))];
1402  list<dag> ret = !if(!eq(P.NumSrcArgs, 3), ret3,
1403                  !if(!eq(P.NumSrcArgs, 2), ret2,
1404                  ret1));
1405}
1406
1407class getVOP3ClampPat<VOPProfile P, SDPatternOperator node> {
1408  list<dag> ret3 = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2, i1:$clamp))];
1409  list<dag> ret2 = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, i1:$clamp))];
1410  list<dag> ret1 = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, i1:$clamp))];
1411  list<dag> ret = !if(!eq(P.NumSrcArgs, 3), ret3,
1412                  !if(!eq(P.NumSrcArgs, 2), ret2,
1413                  ret1));
1414}
1415
1416class getVOP3MAIPat<VOPProfile P, SDPatternOperator node> {
1417  list<dag> mfma_with_abid = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2,
1418                                   timm:$cbsz, timm:$abid, timm:$blgp))];
1419  list<dag> mfma_no_abid = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2,
1420                            timm:$cbsz, timm:$blgp))];
1421
1422  list<dag> ret = !if(!not(P.IsSMFMAC),
1423                      // mfma
1424                      !if(P.HasAbid, mfma_with_abid, mfma_no_abid),
1425
1426                      // smfmac
1427                      [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2, i32:$idx,
1428                                            timm:$cbsz, timm:$abid))]);
1429}
1430
1431class getVOP3MAIScaledPat<VOPProfile P, SDPatternOperator node> {
1432  list<dag> ret = !if(!not(P.IsSMFMAC),
1433                      // mfma
1434                      [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2,
1435                                            timm:$cbsz, timm:$blgp,
1436                                            MFMALdScaleModifierOp:$scale_src0_opsel,
1437                                            i32:$scale_src0,
1438                                            MFMALdScaleModifierOp:$scale_src1_opsel,
1439                                            i32:$scale_src1
1440                                            ))],
1441                      // smfmac
1442                      [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2, i32:$idx,
1443                                            timm:$cbsz, timm:$abid,
1444                                            MFMALdScaleModifierOp:$scale_src0_opsel,
1445                                            i32:$scale_src0,
1446                                            MFMALdScaleModifierOp:$scale_src1_opsel,
1447                                            i32:$scale_src1))]);
1448}
1449
1450class VOP3Features<bit Clamp, bit OpSel, bit Packed, bit MAI> {
1451  bit HasClamp = Clamp;
1452  bit HasOpSel = OpSel;
1453  bit IsPacked = Packed;
1454  bit IsMAI = MAI;
1455}
1456
1457def VOP3_REGULAR         : VOP3Features<0, 0, 0, 0>;
1458def VOP3_CLAMP           : VOP3Features<1, 0, 0, 0>;
1459def VOP3_OPSEL           : VOP3Features<1, 1, 0, 0>;
1460def VOP3_PACKED          : VOP3Features<1, 1, 1, 0>;
1461def VOP3_PACKED_NO_OPSEL : VOP3Features<1, 0, 1, 0>;
1462def VOP3_MAI             : VOP3Features<0, 0, 0, 1>;
1463def VOP3_OPSEL_ONLY      : VOP3Features<0, 1, 0, 0>;
1464
1465// Packed is misleading, but it enables the appropriate op_sel
1466// modifiers.
1467def VOP3P_LD_SCALE : VOP3Features<0, 1, 1, 0>;
1468
1469class VOP3_Profile_Base<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOPProfile<P.ArgVT> {
1470
1471  let HasClamp = !if(Features.HasClamp, 1, P.HasClamp);
1472  let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
1473  let IsMAI    = !if(Features.IsMAI,    1, P.IsMAI);
1474  let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
1475
1476  let HasModifiers =
1477      !if (Features.IsMAI, 0,
1478           !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
1479}
1480
1481class VOP3_Profile<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOP3_Profile_Base<P, Features> {
1482  let IsSingle = 1;
1483
1484}
1485
1486class VOP3_Profile_True16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOPProfile_True16<P> {
1487  let HasClamp = !if(Features.HasClamp, 1, P.HasClamp);
1488  let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
1489  let IsMAI    = !if(Features.IsMAI,    1, P.IsMAI);
1490  let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
1491
1492  let HasModifiers =
1493      !if (Features.IsMAI, 0,
1494           !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
1495  let IsSingle = 1;
1496}
1497
1498class VOP3_Profile_Fake16<VOPProfile P, VOP3Features Features = VOP3_REGULAR> : VOPProfile_Fake16<P> {
1499  let HasClamp = !if(Features.HasClamp, 1, P.HasClamp);
1500  let HasOpSel = !if(Features.HasOpSel, 1, P.HasOpSel);
1501  let IsMAI    = !if(Features.IsMAI,    1, P.IsMAI);
1502  let IsPacked = !if(Features.IsPacked, 1, P.IsPacked);
1503
1504  let HasModifiers =
1505      !if (Features.IsMAI, 0,
1506           !or(Features.IsPacked, Features.HasOpSel, P.HasModifiers));
1507  let IsSingle = 1;
1508}
1509
1510// consistently gives instructions a _e64 suffix
1511multiclass VOP3Inst_Pseudo_Wrapper<string opName, VOPProfile P, list<dag> pattern = [], bit VOP3Only = 0> {
1512    def _e64 : VOP3_Pseudo<opName, P, pattern, VOP3Only>;
1513}
1514
1515class VOP3InstBase<string OpName, VOPProfile P, SDPatternOperator node = null_frag, bit IsVOP2 = 0, bit MAIScaled = false> :
1516  VOP3_Pseudo<OpName, P,
1517    !if(P.HasOpSel,
1518        !if(P.HasModifiers,
1519            getVOP3OpSelModPat<P, node>.ret,
1520            getVOP3OpSelPat<P, node>.ret),
1521        !if(P.HasModifiers,
1522            getVOP3ModPat<P, node>.ret,
1523            !if(IsVOP2,
1524              getVOP3FromVOP2Pat<P, node>.ret,
1525              !if(P.HasIntClamp,
1526                  getVOP3ClampPat<P, node>.ret,
1527                  !if (P.IsMAI,
1528                      !if(MAIScaled, getVOP3MAIScaledPat<P, node>.ret, getVOP3MAIPat<P, node>.ret),
1529                      getVOP3Pat<P, node>.ret))))),
1530    0, P.HasOpSel> {
1531
1532  let IntClamp = P.HasIntClamp;
1533  let AsmMatchConverter =
1534    !if(P.HasOpSel,
1535        "cvtVOP3OpSel",
1536        !if(!or(P.HasModifiers, P.HasOMod, P.HasIntClamp),
1537            "cvtVOP3",
1538            ""));
1539}
1540
1541multiclass VOP3Inst<string OpName, VOPProfile P, SDPatternOperator node = null_frag> {
1542  def _e64 : VOP3InstBase<OpName, P, node>;
1543  let SubtargetPredicate = isGFX11Plus in {
1544    if P.HasExtVOP3DPP then
1545      def _e64_dpp : VOP3_DPP_Pseudo <OpName, P>;
1546  } // end SubtargetPredicate = isGFX11Plus
1547}
1548
1549class UniformUnaryFragOrOp<SDPatternOperator Op> {
1550  SDPatternOperator ret = !if(!or(!isa<SDNode>(Op), !isa<PatFrags>(Op)),
1551                              UniformUnaryFrag<Op>, Op);
1552}
1553
1554multiclass VOP3PseudoScalarInst<string OpName, VOPProfile P,
1555                                SDPatternOperator node = null_frag> {
1556  def _e64 : VOP3_Pseudo<OpName, P, [(set P.DstVT:$vdst,
1557      (UniformUnaryFragOrOp<node>.ret
1558          (P.Src0VT (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers, i1:$clamp,
1559                               i32:$omod))))]>;
1560}
1561
1562multiclass VOP3Inst_t16_with_profiles<string OpName, VOPProfile P, VOPProfile P_t16,
1563                        VOPProfile P_fake16,
1564                        SDPatternOperator node = null_frag,
1565                        SDPatternOperator node_t16 = node> {
1566  let True16Predicate = NotHasTrue16BitInsts  in {
1567    defm NAME : VOP3Inst<OpName, P, node>;
1568  }
1569  let True16Predicate = UseRealTrue16Insts in {
1570    defm _t16 : VOP3Inst<OpName # "_t16", P_t16, node_t16>;
1571  }
1572  let True16Predicate = UseFakeTrue16Insts in {
1573    defm _fake16 : VOP3Inst<OpName # "_fake16", P_fake16, node>;
1574  }
1575}
1576
1577multiclass VOP3Inst_t16<string OpName, VOPProfile P,
1578                        SDPatternOperator node = null_frag,
1579                        SDPatternOperator node_t16 = node>
1580 : VOP3Inst_t16_with_profiles<OpName, VOP3_Profile<P, VOP3_OPSEL>,
1581                VOP3_Profile_True16<P, VOP3_OPSEL>, VOP3_Profile_Fake16<P, VOP3_OPSEL>,
1582                node, node_t16>;
1583
1584//===----------------------------------------------------------------------===//
1585// VOP3 DPP
1586//===----------------------------------------------------------------------===//
1587
1588class VOP3_DPP16_Helper<bits<10> op, VOP_DPP_Pseudo ps, string opName = ps.OpName>
1589    : VOP3_DPP<op, opName, ps.Pfl, 1> {
1590  let VOP3_OPSEL = ps.Pfl.HasOpSel;
1591  let IsDOT = ps.IsDOT;
1592  let hasSideEffects = ps.hasSideEffects;
1593  let Defs = ps.Defs;
1594  let SchedRW = ps.SchedRW;
1595  let Uses = ps.Uses;
1596  let AssemblerPredicate = HasDPP16;
1597  let SubtargetPredicate = ps.SubtargetPredicate;
1598  let OtherPredicates = ps.OtherPredicates;
1599}
1600
1601class VOP3_DPP16_t16_Helper<bits<10> op, VOP_DPP_Pseudo ps,
1602                          string opName = ps.OpName>
1603    : VOP3_DPP_t16<op, opName, ps.Pfl, 1> {
1604  let VOP3_OPSEL = ps.Pfl.HasOpSel;
1605  let IsDOT = ps.IsDOT;
1606  let hasSideEffects = ps.hasSideEffects;
1607  let Defs = ps.Defs;
1608  let SchedRW = ps.SchedRW;
1609  let Uses = ps.Uses;
1610  let AssemblerPredicate = HasDPP16;
1611  let SubtargetPredicate = ps.SubtargetPredicate;
1612  let OtherPredicates = ps.OtherPredicates;
1613}
1614
1615class VOP3_DPP16<bits<10> op, VOP_DPP_Pseudo ps, int subtarget,
1616                 string opName = ps.OpName>
1617    : VOP3_DPP16_Helper<op, ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>;
1618
1619class VOP3_DPP16_t16<bits<10> op, VOP_DPP_Pseudo ps, int subtarget,
1620                     string opName = ps.OpName>
1621    : VOP3_DPP16_t16_Helper<op, ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>;
1622
1623class VOP3_DPP16_Gen<bits<10> op, VOP_DPP_Pseudo ps, GFXGen Gen,
1624                     string opName = ps.OpName>
1625    : VOP3_DPP16<op, ps, Gen.Subtarget, opName> {
1626  let AssemblerPredicate = Gen.AssemblerPredicate;
1627  let DecoderNamespace = Gen.DecoderNamespace;
1628}
1629
1630class VOP3_DPP16_Gen_t16<bits<10> op, VOP_DPP_Pseudo ps, GFXGen Gen,
1631                         string opName = ps.OpName>
1632    : VOP3_DPP16_t16<op, ps, Gen.Subtarget, opName> {
1633  let True16Predicate =
1634      !if (ps.Pfl.IsRealTrue16, UseRealTrue16Insts, NoTrue16Predicate);
1635  let AssemblerPredicate = Gen.AssemblerPredicate;
1636  let DecoderNamespace =
1637      Gen.DecoderNamespace #!if (ps.Pfl.IsRealTrue16, "", "_FAKE16");
1638}
1639
1640class Base_VOP3_DPP8<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
1641    : VOP3_DPP8<op, opName, ps.Pfl> {
1642  let VOP3_OPSEL = ps.Pfl.HasOpSel;
1643  let IsDOT = ps.IsDOT;
1644  let hasSideEffects = ps.hasSideEffects;
1645  let Defs = ps.Defs;
1646  let SchedRW = ps.SchedRW;
1647  let Uses = ps.Uses;
1648
1649  let SubtargetPredicate = ps.SubtargetPredicate;
1650  let OtherPredicates = ps.OtherPredicates;
1651  let True16Predicate = ps.True16Predicate;
1652}
1653
1654class Base_VOP3_DPP8_t16<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
1655    : VOP3_DPP8_t16<op, opName, ps.Pfl> {
1656  let VOP3_OPSEL = ps.Pfl.HasOpSel;
1657  let IsDOT = ps.IsDOT;
1658  let hasSideEffects = ps.hasSideEffects;
1659  let Defs = ps.Defs;
1660  let SchedRW = ps.SchedRW;
1661  let Uses = ps.Uses;
1662
1663  let OtherPredicates = ps.OtherPredicates;
1664  let True16Predicate = ps.True16Predicate;
1665}
1666
1667class Base_VOP3b_DPP16<bits<10> op, VOP_DPP_Pseudo ps,
1668                       string opName = ps.OpName>
1669    : VOP3_DPP16_Helper<op, ps, opName> {
1670  bits<7> sdst;
1671  let Inst{14 - 8} = sdst;
1672}
1673
1674class VOP3b_DPP8_Base<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
1675    : Base_VOP3_DPP8<op, ps, opName> {
1676  bits<7> sdst;
1677  let Inst{14 - 8} = sdst;
1678}
1679
1680class VOP3b_DPP8_Base_t16<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
1681    : Base_VOP3_DPP8<op, ps, opName> {
1682  bits<8> sdst;
1683  let Inst{14 - 8} = sdst{7-1};
1684}
1685
1686//===----------------------------------------------------------------------===//
1687// VOP3 GFX11, GFX12
1688//===----------------------------------------------------------------------===//
1689
1690multiclass VOP3_Real_Base<GFXGen Gen, bits<10> op, string opName = NAME,
1691                          bit isSingle = 0> {
1692  defvar ps = !cast<VOP_Pseudo>(opName#"_e64");
1693  let IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {
1694    if ps.Pfl.IsFP8SrcByteSel then {
1695      def _e64#Gen.Suffix :
1696        VOP3_Real_Gen<ps, Gen>,
1697        VOP3FP8OpSel_src_bytesel_gfx11_gfx12<op, ps.Pfl>;
1698    } else if ps.Pfl.IsFP8DstByteSel then {
1699      def _e64#Gen.Suffix :
1700        VOP3_Real_Gen<ps, Gen>,
1701        VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<op, ps.Pfl>;
1702    } else if ps.Pfl.HasOpSel then {
1703      def _e64#Gen.Suffix :
1704        VOP3_Real_Gen<ps, Gen>,
1705        VOP3OpSel_gfx11_gfx12<op, ps.Pfl>;
1706    } else {
1707      def _e64#Gen.Suffix :
1708        VOP3_Real_Gen<ps, Gen>,
1709        VOP3e_gfx11_gfx12<op, ps.Pfl>;
1710    }
1711  }
1712}
1713
1714multiclass VOP3Dot_Real_Base<GFXGen Gen, bits<10> op, string asmName, string opName = NAME,
1715                             bit isSingle = 0> {
1716  defvar ps = !cast<VOP_Pseudo>(opName#"_e64");
1717  let AsmString = asmName # ps.AsmOperands,
1718      DecoderNamespace = Gen.DecoderNamespace # !if(ps.Pfl.IsRealTrue16, "", "_FAKE16"),
1719      IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {
1720    def _e64#Gen.Suffix :
1721      VOP3_Real_Gen<ps, Gen>,
1722      VOP3DotOpSel_gfx11_gfx12<op, ps.Pfl>;
1723  }
1724}
1725
1726multiclass VOP3_Real_with_name<GFXGen Gen, bits<10> op, string opName,
1727                               string asmName, string pseudo_mnemonic = "", bit isSingle = 0> {
1728  defvar ps = !cast<VOP_Pseudo>(opName#"_e64");
1729  let AsmString = asmName # ps.AsmOperands,
1730      IsSingle = !or(isSingle, ps.Pfl.IsSingle) in {
1731    // FIXME-TRUE16 support FP8 instructions properly
1732    if ps.Pfl.IsFP8SrcByteSel then {
1733      def _e64#Gen.Suffix :
1734        VOP3_Real_Gen<ps, Gen>,
1735        VOP3FP8OpSel_src_bytesel_gfx11_gfx12<op, ps.Pfl>;
1736    } else if ps.Pfl.IsFP8DstByteSel then {
1737      def _e64#Gen.Suffix :
1738        VOP3_Real_Gen<ps, Gen>,
1739        VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<op, ps.Pfl>;
1740    } else {
1741      if ps.Pfl.IsRealTrue16 then {
1742        def _e64#Gen.Suffix :
1743          VOP3_Real_Gen<ps, Gen>,
1744          VOP3e_t16_gfx11_gfx12<op, ps.Pfl>;
1745      } else {
1746        if ps.Pfl.HasOpSel then {
1747          def _e64#Gen.Suffix :
1748            VOP3_Real_Gen<ps, Gen>,
1749            VOP3OpSel_gfx11_gfx12<op, ps.Pfl>;
1750        } else {
1751          def _e64#Gen.Suffix :
1752            VOP3_Real_Gen<ps, Gen>,
1753            VOP3e_gfx11_gfx12<op, ps.Pfl>;
1754        }
1755      }
1756    }
1757  }
1758  def Gen.Suffix#"_VOP3_alias" : LetDummies,
1759                                 AMDGPUMnemonicAlias<!if(!empty(pseudo_mnemonic),
1760                                                     ps.Mnemonic, pseudo_mnemonic), asmName, ""> {
1761    let AssemblerPredicate = Gen.AssemblerPredicate;
1762  }
1763}
1764
1765// for READLANE/WRITELANE
1766multiclass VOP3_Real_No_Suffix<GFXGen Gen, bits<10> op, string opName = NAME> {
1767  defvar ps = !cast<VOP_Pseudo>(opName);
1768    def _e64#Gen.Suffix :
1769      VOP3_Real_Gen<ps, Gen>,
1770      VOP3e_gfx11_gfx12<op, ps.Pfl>;
1771}
1772
1773multiclass VOP3_Real_dpp_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
1774  defvar ps = !cast<VOP_DPP_Pseudo>(opName#"_e64"#"_dpp");
1775  if ps.Pfl.IsTrue16 then
1776    def _e64_dpp#Gen.Suffix :
1777      VOP3_DPP16_Gen_t16<op, ps, Gen>;
1778  else
1779    def _e64_dpp#Gen.Suffix :
1780      VOP3_DPP16_Gen<op, ps, Gen>;
1781}
1782
1783multiclass VOP3Dot_Real_dpp_Base<GFXGen Gen, bits<10> op, string asmName, string opName = NAME> {
1784  defvar ps = !cast<VOP_DPP_Pseudo>(opName#"_e64"#"_dpp");
1785  def _e64_dpp#Gen.Suffix :
1786    VOP3_DPP16_Gen_t16<op, ps, Gen> {
1787      let AsmString = asmName # ps.Pfl.AsmVOP3DPP16;
1788      let DecoderNamespace = Gen.DecoderNamespace
1789                             # !if(ps.Pfl.IsRealTrue16, "", "_FAKE16");
1790      let Inst{11} = ?;
1791      let Inst{12} = ?;
1792    }
1793}
1794
1795multiclass VOP3_Real_dpp_with_name<GFXGen Gen, bits<10> op, string opName,
1796                                   string asmName> {
1797  defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
1798  let AsmString = asmName # ps.Pfl.AsmVOP3DPP16 in {
1799    defm NAME : VOP3_Real_dpp_Base<Gen, op, opName>;
1800  }
1801}
1802
1803multiclass VOP3_Real_dpp8_Base<GFXGen Gen, bits<10> op, string opName = NAME> {
1804  defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
1805  def _e64_dpp8#Gen.Suffix : Base_VOP3_DPP8<op, ps> {
1806    let DecoderNamespace = Gen.DecoderNamespace;
1807    let AssemblerPredicate = Gen.AssemblerPredicate;
1808  }
1809}
1810
1811multiclass VOP3Dot_Real_dpp8_Base<GFXGen Gen, bits<10> op, string asmName, string opName = NAME> {
1812  defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
1813  def _e64_dpp8#Gen.Suffix : Base_VOP3_DPP8_t16<op, ps> {
1814    let Inst{11} = ?;
1815    let Inst{12} = ?;
1816    let AsmString = asmName # ps.Pfl.AsmVOP3DPP8;
1817    let DecoderNamespace = Gen.DecoderNamespace
1818                           # !if(ps.Pfl.IsRealTrue16, "", "_FAKE16");
1819    let AssemblerPredicate = Gen.AssemblerPredicate;
1820  }
1821}
1822
1823multiclass VOP3_Real_dpp8_with_name<GFXGen Gen, bits<10> op, string opName,
1824                                    string asmName> {
1825  defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
1826  let AsmString = asmName # ps.Pfl.AsmVOP3DPP8,
1827      DecoderNamespace = Gen.DecoderNamespace#
1828                         !if(ps.Pfl.IsRealTrue16, "", "_FAKE16"),
1829      True16Predicate = !if(ps.Pfl.IsRealTrue16, UseRealTrue16Insts,
1830                            NoTrue16Predicate) in {
1831    defm NAME : VOP3_Real_dpp8_Base<Gen, op, opName>;
1832  }
1833}
1834
1835multiclass VOP3be_Real<GFXGen Gen, bits<10> op, string opName, string asmName,
1836                      bit isSingle = 0> {
1837  defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
1838  let IsSingle = !or(isSingle, ps.Pfl.IsSingle) in
1839    def _e64#Gen.Suffix :
1840      VOP3_Real_Gen<ps, Gen, asmName>,
1841      VOP3be_gfx11_gfx12<op, ps.Pfl> ;
1842}
1843
1844multiclass VOP3be_Real_dpp<GFXGen Gen, bits<10> op, string opName,
1845                           string asmName> {
1846  defvar ps = !cast<VOP3_Pseudo>(opName #"_e64");
1847  defvar dpp_ps = !cast<VOP_DPP_Pseudo>(opName #"_e64" #"_dpp");
1848  def _e64_dpp#Gen.Suffix : Base_VOP3b_DPP16<op, dpp_ps, asmName>,
1849                            SIMCInstr<dpp_ps.PseudoInstr, Gen.Subtarget> {
1850    let DecoderNamespace = Gen.DecoderNamespace;
1851    let AssemblerPredicate = Gen.AssemblerPredicate;
1852  }
1853}
1854
1855multiclass VOP3be_Real_dpp8<GFXGen Gen, bits<10> op, string opName,
1856                            string asmName> {
1857  defvar ps = !cast<VOP3_Pseudo>(opName #"_e64");
1858  def _e64_dpp8#Gen.Suffix : VOP3b_DPP8_Base<op, ps, asmName> {
1859    let DecoderNamespace = Gen.DecoderNamespace;
1860    let AssemblerPredicate = Gen.AssemblerPredicate;
1861  }
1862}
1863
1864// VOP1 and VOP2 depend on these triple defs
1865multiclass VOP3_Realtriple<GFXGen Gen, bits<10> op, bit isSingle = 0,
1866                           string opName = NAME> :
1867  VOP3_Real_Base<Gen, op, opName, isSingle>,
1868  VOP3_Real_dpp_Base<Gen, op, opName>,
1869  VOP3_Real_dpp8_Base<Gen, op, opName>;
1870
1871multiclass VOP3Dot_Realtriple<GFXGen Gen, bits<10> op, string asmName, bit isSingle = 0,
1872                              string opName = NAME> :
1873  VOP3Dot_Real_Base<Gen, op, asmName, opName, isSingle>,
1874  VOP3Dot_Real_dpp_Base<Gen, op, asmName, opName>,
1875  VOP3Dot_Real_dpp8_Base<Gen, op, asmName, opName>;
1876
1877multiclass VOP3Only_Realtriple<GFXGen Gen, bits<10> op> :
1878  VOP3_Realtriple<Gen, op, 1>;
1879
1880multiclass VOP3_Realtriple_with_name<GFXGen Gen, bits<10> op, string opName,
1881                                     string asmName, string pseudo_mnemonic = "", bit isSingle = 0> :
1882  VOP3_Real_with_name<Gen, op, opName, asmName, pseudo_mnemonic, isSingle>,
1883  VOP3_Real_dpp_with_name<Gen, op, opName, asmName>,
1884  VOP3_Real_dpp8_with_name<Gen, op, opName, asmName>;
1885
1886multiclass VOP3Only_Realtriple_with_name<GFXGen Gen, bits<10> op, string opName,
1887                                         string asmName> :
1888  VOP3_Realtriple_with_name<Gen, op, opName, asmName, "", 1>;
1889
1890multiclass VOP3be_Realtriple<
1891    GFXGen Gen, bits<10> op, bit isSingle = 0, string opName = NAME,
1892    string asmName = !cast<VOP_Pseudo>(opName#"_e64").Mnemonic> :
1893  VOP3be_Real<Gen, op, opName, asmName, isSingle>,
1894  VOP3be_Real_dpp<Gen, op, opName, asmName>,
1895  VOP3be_Real_dpp8<Gen, op, opName, asmName>;
1896
1897multiclass VOP3beOnly_Realtriple<GFXGen Gen, bits<10> op> :
1898  VOP3be_Realtriple<Gen, op, 1>;
1899
1900//===----------------------------------------------------------------------===//
1901// VOP3 GFX11
1902//===----------------------------------------------------------------------===//
1903
1904// VOP1 and VOP2 depend on these triple defs
1905
1906multiclass VOP3_Realtriple_t16_gfx11<bits<10> op, string asmName, string opName = NAME,
1907                                     string pseudo_mnemonic = "", bit isSingle = 0> :
1908  VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
1909
1910multiclass VOP3_Realtriple_t16_and_fake16_gfx11<bits<10> op, string asmName, string opName = NAME,
1911                                                string pseudo_mnemonic = "", bit isSingle = 0> {
1912  defm _t16: VOP3_Realtriple_t16_gfx11<op, asmName, opName#"_t16", pseudo_mnemonic, isSingle>;
1913  defm _fake16: VOP3_Realtriple_t16_gfx11<op, asmName, opName#"_fake16", pseudo_mnemonic, isSingle>;
1914}
1915
1916multiclass VOP3Only_Realtriple_t16_gfx11<bits<10> op, string asmName,
1917                                     string opName = NAME, string pseudo_mnemonic = "">
1918  : VOP3_Realtriple_t16_gfx11<op, asmName, opName, pseudo_mnemonic, 1>;
1919
1920multiclass VOP3Only_Realtriple_t16_and_fake16_gfx11<bits<10> op, string asmName,
1921                                                    string opName = NAME, string pseudo_mnemonic = ""> {
1922  defm _t16: VOP3_Realtriple_t16_gfx11<op, asmName, opName#"_t16", pseudo_mnemonic, 1>;
1923  defm _fake16: VOP3_Realtriple_t16_gfx11<op, asmName, opName#"_fake16", pseudo_mnemonic, 1>;
1924}
1925
1926multiclass VOP3be_Real_gfx11<bits<10> op, string opName, string asmName,
1927                             bit isSingle = 0> :
1928  VOP3be_Real<GFX11Gen, op, opName, asmName, isSingle>;
1929
1930multiclass VOP3_Real_Base_gfx11<bits<10> op, string opName = NAME,
1931                                bit isSingle = 0> :
1932  VOP3_Real_Base<GFX11Gen, op, opName, isSingle>;
1933
1934multiclass VOP3_Realtriple_gfx11<bits<10> op, bit isSingle = 0,
1935                                 string opName = NAME> :
1936  VOP3_Realtriple<GFX11Gen, op, isSingle, opName>;
1937
1938//===----------------------------------------------------------------------===//
1939// VOP3 GFX12
1940//===----------------------------------------------------------------------===//
1941
1942multiclass VOP3Only_Realtriple_gfx12<bits<10> op, bit isSingle = 0> :
1943  VOP3_Realtriple<GFX12Gen, op, isSingle>;
1944
1945// IsSingle is captured from the vopprofile for these instructions, but the
1946// following alternative is more explicit
1947multiclass VOP3Only_Real_Base_gfx12<bits<10> op> :
1948  VOP3_Real_Base<GFX12Gen, op, NAME, 1/*IsSingle*/>;
1949
1950multiclass VOP3_Realtriple_t16_gfx12<bits<10> op, string asmName, string opName = NAME,
1951                                     string pseudo_mnemonic = "", bit isSingle = 0> :
1952  VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
1953
1954multiclass VOP3_Realtriple_t16_and_fake16_gfx12<bits<10> op, string asmName, string opName = NAME,
1955                                                string pseudo_mnemonic = "", bit isSingle = 0> {
1956  defm _t16:VOP3_Realtriple_t16_gfx12<op, asmName, opName#"_t16", pseudo_mnemonic, isSingle>;
1957  defm _fake16:VOP3_Realtriple_t16_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic, isSingle>;
1958}
1959
1960multiclass VOP3Only_Realtriple_t16_gfx12<bits<10> op, string asmName,
1961                                     string opName = NAME, string pseudo_mnemonic = "">
1962  : VOP3_Realtriple_t16_gfx12<op, asmName, opName, pseudo_mnemonic, 1>;
1963
1964multiclass VOP3Only_Realtriple_t16_and_fake16_gfx12<bits<10> op, string asmName,
1965                                     string opName = NAME, string pseudo_mnemonic = ""> {
1966  defm _t16 : VOP3Only_Realtriple_t16_gfx12<op, asmName, opName#"_t16", pseudo_mnemonic>;
1967  defm _fake16 : VOP3Only_Realtriple_t16_gfx12<op, asmName, opName#"_fake16", pseudo_mnemonic>;
1968}
1969
1970multiclass VOP3be_Real_with_name_gfx12<bits<10> op, string opName,
1971                                       string asmName, bit isSingle = 0> {
1972  defvar ps = !cast<VOP3_Pseudo>(opName#"_e64");
1973  let AsmString = asmName # ps.AsmOperands,
1974      IsSingle = !or(isSingle, ps.Pfl.IsSingle) in
1975    def _e64_gfx12 :
1976      VOP3_Real_Gen<ps, GFX12Gen, asmName>,
1977      VOP3be_gfx11_gfx12<op, ps.Pfl>;
1978  def : AMDGPUMnemonicAlias<ps.Mnemonic, asmName> {
1979    let AssemblerPredicate = GFX12Gen.AssemblerPredicate;
1980  }
1981}
1982
1983multiclass VOP3_Realtriple_with_name_gfx12<bits<10> op, string opName,
1984                                           string asmName, string pseudo_mnemonic = "", bit isSingle = 0> :
1985  VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
1986
1987multiclass VOP3Only_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,
1988                                                     string asmName> :
1989  VOP3Only_Realtriple_with_name<GFX11Gen, op, opName, asmName>,
1990  VOP3Only_Realtriple_with_name<GFX12Gen, op, opName, asmName>;
1991
1992multiclass VOP3Dot_Realtriple_gfx11_gfx12<bits<10> op, string asmName, bit isSingle = 0,
1993                                          string opName = NAME> :
1994  VOP3Dot_Realtriple<GFX11Gen, op, asmName, isSingle, opName>,
1995  VOP3Dot_Realtriple<GFX12Gen, op, asmName, isSingle, opName>;
1996
1997
1998//===----------------------------------------------------------------------===//
1999
2000include "VOPCInstructions.td"
2001include "VOP1Instructions.td"
2002include "VOP2Instructions.td"
2003include "VOP3Instructions.td"
2004include "VOP3PInstructions.td"
2005include "VOPDInstructions.td"
2006
2007class ClassPat<Instruction inst, ValueType vt> : GCNPat <
2008  (is_fpclass (vt (VOP3ModsNonCanonicalizing vt:$src0, i32:$src0_mods)), (i32 timm:$mask)),
2009  (inst i32:$src0_mods, vt:$src0, (V_MOV_B32_e32 timm:$mask))
2010>;
2011
2012class ClassPat_t16<Instruction inst, ValueType vt> : GCNPat <
2013  (is_fpclass (vt (VOP3ModsNonCanonicalizing vt:$src0, i32:$src0_mods)), (i32 timm:$mask)),
2014  (inst i32:$src0_mods, vt:$src0, SRCMODS.NONE, (V_MOV_B32_e32 timm:$mask))
2015>;
2016
2017def : ClassPat<V_CMP_CLASS_F16_e64, f16> {
2018  let OtherPredicates = [Has16BitInsts];
2019  let True16Predicate = NotHasTrue16BitInsts;
2020}
2021
2022def : ClassPat_t16<V_CMP_CLASS_F16_fake16_e64, f16> {
2023  let True16Predicate = UseFakeTrue16Insts;
2024}
2025
2026def : ClassPat<V_CMP_CLASS_F32_e64, f32>;
2027def : ClassPat<V_CMP_CLASS_F64_e64, f64>;
2028
2029class VOPInfoTable <string Format> : GenericTable {
2030  let FilterClass = Format # "_Real";
2031  let CppTypeName = "VOPInfo";
2032  let Fields = ["Opcode", "IsSingle"];
2033
2034  let PrimaryKey = ["Opcode"];
2035  let PrimaryKeyName = "get" # Format # "OpcodeHelper";
2036}
2037
2038def VOP1InfoTable : VOPInfoTable<"VOP1">;
2039def VOP2InfoTable : VOPInfoTable<"VOP2">;
2040def VOP3InfoTable : VOPInfoTable<"VOP3">;
2041
2042class VOPC64Table <string Format> : GenericTable {
2043  let FilterClass = "VOPC64_" # Format;
2044  let CppTypeName = "VOPC64DPPInfo";
2045  let Fields = ["Opcode"];
2046
2047  let PrimaryKey = ["Opcode"];
2048  let PrimaryKeyName = "isVOPC64" # Format # "OpcodeHelper";
2049}
2050
2051def VOPC64DPPTable : VOPC64Table<"DPP">;
2052def VOPC64DPP8Table : VOPC64Table<"DPP8">;
2053
2054class AsmOnlyInfoTable <string Format, string Class>: GenericTable {
2055  let FilterClass = Class;
2056  let FilterClassField = "isAsmParserOnly";
2057  let CppTypeName = Format # "DPPAsmOnlyInfo";
2058  let Fields = ["Opcode"];
2059
2060  let PrimaryKey = ["Opcode"];
2061  let PrimaryKeyName = "is" # Format # "AsmOnlyOpcodeHelper";
2062}
2063
2064def VOPCAsmOnlyInfoTable : AsmOnlyInfoTable <"VOPC", "VOPC_DPPe_Common">;
2065
2066def VOPTrue16Table : GenericTable {
2067  let FilterClass = "VOP_Pseudo";
2068  let CppTypeName = "VOPTrue16Info";
2069  let Fields = ["Opcode", "IsTrue16"];
2070
2071  let PrimaryKey = ["Opcode"];
2072  let PrimaryKeyName = "getTrue16OpcodeHelper";
2073}
2074