xref: /netbsd-src/external/apache2/llvm/dist/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h (revision 82d56013d7b633d116a93943de88e08335357a7c) 
1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file declares the API for the instruction selector.
10 /// This class is responsible for selecting machine instructions.
11 /// It's implemented by the target. It's used by the InstructionSelect pass.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
21 #include "llvm/CodeGen/GlobalISel/Utils.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/CodeGen/TargetOpcodes.h"
27 #include "llvm/CodeGen/TargetRegisterInfo.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/raw_ostream.h"
33 #include <cassert>
34 #include <cstddef>
35 #include <cstdint>
36 
37 namespace llvm {
38 
39 /// GlobalISel PatFrag Predicates
40 enum {
41   GIPFP_I64_Invalid = 0,
42   GIPFP_APInt_Invalid = 0,
43   GIPFP_APFloat_Invalid = 0,
44   GIPFP_MI_Invalid = 0,
45 };
46 
47 template <class TgtInstructionSelector, class PredicateBitset,
48           class ComplexMatcherMemFn, class CustomRendererFn>
executeMatchTable(TgtInstructionSelector & ISel,NewMIVector & OutMIs,MatcherState & State,const ISelInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ISelInfo,const int64_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage & CoverageInfo)49 bool InstructionSelector::executeMatchTable(
50     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52         &ISelInfo,
53     const int64_t *MatchTable, const TargetInstrInfo &TII,
54     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56     CodeGenCoverage &CoverageInfo) const {
57 
58   uint64_t CurrentIdx = 0;
59   SmallVector<uint64_t, 4> OnFailResumeAt;
60 
61   // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
62   bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
63 
64   const uint16_t Flags = State.MIs[0]->getFlags();
65 
66   enum RejectAction { RejectAndGiveUp, RejectAndResume };
67   auto handleReject = [&]() -> RejectAction {
68     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69                     dbgs() << CurrentIdx << ": Rejected\n");
70     if (OnFailResumeAt.empty())
71       return RejectAndGiveUp;
72     CurrentIdx = OnFailResumeAt.pop_back_val();
73     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
74                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
75                            << OnFailResumeAt.size() << " try-blocks remain)\n");
76     return RejectAndResume;
77   };
78 
79   auto propagateFlags = [=](NewMIVector &OutMIs) {
80     for (auto MIB : OutMIs) {
81       // Set the NoFPExcept flag when no original matched instruction could
82       // raise an FP exception, but the new instruction potentially might.
83       uint16_t MIBFlags = Flags;
84       if (NoFPException && MIB->mayRaiseFPException())
85         MIBFlags |= MachineInstr::NoFPExcept;
86       MIB.setMIFlags(MIBFlags);
87     }
88 
89     return true;
90   };
91 
92   while (true) {
93     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
94     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
95     switch (MatcherOpcode) {
96     case GIM_Try: {
97       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
98                       dbgs() << CurrentIdx << ": Begin try-block\n");
99       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
100       break;
101     }
102 
103     case GIM_RecordInsn: {
104       int64_t NewInsnID = MatchTable[CurrentIdx++];
105       int64_t InsnID = MatchTable[CurrentIdx++];
106       int64_t OpIdx = MatchTable[CurrentIdx++];
107 
108       // As an optimisation we require that MIs[0] is always the root. Refuse
109       // any attempt to modify it.
110       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
111 
112       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
113       if (!MO.isReg()) {
114         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
115                         dbgs() << CurrentIdx << ": Not a register\n");
116         if (handleReject() == RejectAndGiveUp)
117           return false;
118         break;
119       }
120       if (Register::isPhysicalRegister(MO.getReg())) {
121         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
122                         dbgs() << CurrentIdx << ": Is a physical register\n");
123         if (handleReject() == RejectAndGiveUp)
124           return false;
125         break;
126       }
127 
128       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
129       if ((size_t)NewInsnID < State.MIs.size())
130         State.MIs[NewInsnID] = NewMI;
131       else {
132         assert((size_t)NewInsnID == State.MIs.size() &&
133                "Expected to store MIs in order");
134         State.MIs.push_back(NewMI);
135       }
136       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
137                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
138                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
139                              << ")\n");
140       break;
141     }
142 
143     case GIM_CheckFeatures: {
144       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
145       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146                       dbgs() << CurrentIdx
147                              << ": GIM_CheckFeatures(ExpectedBitsetID="
148                              << ExpectedBitsetID << ")\n");
149       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
150           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
151         if (handleReject() == RejectAndGiveUp)
152           return false;
153       }
154       break;
155     }
156 
157     case GIM_CheckOpcode:
158     case GIM_CheckOpcodeIsEither: {
159       int64_t InsnID = MatchTable[CurrentIdx++];
160       int64_t Expected0 = MatchTable[CurrentIdx++];
161       int64_t Expected1 = -1;
162       if (MatcherOpcode == GIM_CheckOpcodeIsEither)
163         Expected1 = MatchTable[CurrentIdx++];
164 
165       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
166       unsigned Opcode = State.MIs[InsnID]->getOpcode();
167 
168       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
169         dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
170         << "], ExpectedOpcode=" << Expected0;
171         if (MatcherOpcode == GIM_CheckOpcodeIsEither)
172           dbgs() << " || " << Expected1;
173         dbgs() << ") // Got=" << Opcode << "\n";
174       );
175 
176       if (Opcode != Expected0 && Opcode != Expected1) {
177         if (handleReject() == RejectAndGiveUp)
178           return false;
179       }
180       break;
181     }
182     case GIM_SwitchOpcode: {
183       int64_t InsnID = MatchTable[CurrentIdx++];
184       int64_t LowerBound = MatchTable[CurrentIdx++];
185       int64_t UpperBound = MatchTable[CurrentIdx++];
186       int64_t Default = MatchTable[CurrentIdx++];
187 
188       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
189       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
190 
191       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
192         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
193                << LowerBound << ", " << UpperBound << "), Default=" << Default
194                << ", JumpTable...) // Got=" << Opcode << "\n";
195       });
196       if (Opcode < LowerBound || UpperBound <= Opcode) {
197         CurrentIdx = Default;
198         break;
199       }
200       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
201       if (!CurrentIdx) {
202         CurrentIdx = Default;
203         break;
204       }
205       OnFailResumeAt.push_back(Default);
206       break;
207     }
208 
209     case GIM_SwitchType: {
210       int64_t InsnID = MatchTable[CurrentIdx++];
211       int64_t OpIdx = MatchTable[CurrentIdx++];
212       int64_t LowerBound = MatchTable[CurrentIdx++];
213       int64_t UpperBound = MatchTable[CurrentIdx++];
214       int64_t Default = MatchTable[CurrentIdx++];
215 
216       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
217       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
218 
219       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
220         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
221                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
222                << UpperBound << "), Default=" << Default
223                << ", JumpTable...) // Got=";
224         if (!MO.isReg())
225           dbgs() << "Not a VReg\n";
226         else
227           dbgs() << MRI.getType(MO.getReg()) << "\n";
228       });
229       if (!MO.isReg()) {
230         CurrentIdx = Default;
231         break;
232       }
233       const LLT Ty = MRI.getType(MO.getReg());
234       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
235       if (TyI == ISelInfo.TypeIDMap.end()) {
236         CurrentIdx = Default;
237         break;
238       }
239       const int64_t TypeID = TyI->second;
240       if (TypeID < LowerBound || UpperBound <= TypeID) {
241         CurrentIdx = Default;
242         break;
243       }
244       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
245       if (!CurrentIdx) {
246         CurrentIdx = Default;
247         break;
248       }
249       OnFailResumeAt.push_back(Default);
250       break;
251     }
252 
253     case GIM_CheckNumOperands: {
254       int64_t InsnID = MatchTable[CurrentIdx++];
255       int64_t Expected = MatchTable[CurrentIdx++];
256       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
257                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
258                              << InsnID << "], Expected=" << Expected << ")\n");
259       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
260       if (State.MIs[InsnID]->getNumOperands() != Expected) {
261         if (handleReject() == RejectAndGiveUp)
262           return false;
263       }
264       break;
265     }
266     case GIM_CheckI64ImmPredicate:
267     case GIM_CheckImmOperandPredicate: {
268       int64_t InsnID = MatchTable[CurrentIdx++];
269       int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate
270                           ? MatchTable[CurrentIdx++]
271                           : 1;
272       int64_t Predicate = MatchTable[CurrentIdx++];
273       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
274                       dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
275                              << InsnID << "]->getOperand(" << OpIdx
276                              << "), Predicate=" << Predicate << ")\n");
277       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
278       assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
279               State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
280              "Expected immediate operand");
281       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
282       int64_t Value = 0;
283       if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
284         Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
285       else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
286         Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
287       else
288         llvm_unreachable("Expected Imm or CImm operand");
289 
290       if (!testImmPredicate_I64(Predicate, Value))
291         if (handleReject() == RejectAndGiveUp)
292           return false;
293       break;
294     }
295     case GIM_CheckAPIntImmPredicate: {
296       int64_t InsnID = MatchTable[CurrentIdx++];
297       int64_t Predicate = MatchTable[CurrentIdx++];
298       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
299                       dbgs()
300                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
301                           << InsnID << "], Predicate=" << Predicate << ")\n");
302       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
303       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
304              "Expected G_CONSTANT");
305       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
306       APInt Value;
307       if (State.MIs[InsnID]->getOperand(1).isCImm())
308         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
309       else
310         llvm_unreachable("Expected Imm or CImm operand");
311 
312       if (!testImmPredicate_APInt(Predicate, Value))
313         if (handleReject() == RejectAndGiveUp)
314           return false;
315       break;
316     }
317     case GIM_CheckAPFloatImmPredicate: {
318       int64_t InsnID = MatchTable[CurrentIdx++];
319       int64_t Predicate = MatchTable[CurrentIdx++];
320       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
321                       dbgs()
322                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
323                           << InsnID << "], Predicate=" << Predicate << ")\n");
324       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
325       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
326              "Expected G_FCONSTANT");
327       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
328       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
329       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
330 
331       if (!testImmPredicate_APFloat(Predicate, Value))
332         if (handleReject() == RejectAndGiveUp)
333           return false;
334       break;
335     }
336     case GIM_CheckIsBuildVectorAllOnes:
337     case GIM_CheckIsBuildVectorAllZeros: {
338       int64_t InsnID = MatchTable[CurrentIdx++];
339 
340       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341                       dbgs() << CurrentIdx
342                              << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
343                              << InsnID << "])\n");
344       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345 
346       const MachineInstr *MI = State.MIs[InsnID];
347       assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
348               MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
349              "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
350 
351       if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
352         if (!isBuildVectorAllOnes(*MI, MRI)) {
353           if (handleReject() == RejectAndGiveUp)
354             return false;
355         }
356       } else {
357         if (!isBuildVectorAllZeros(*MI, MRI)) {
358           if (handleReject() == RejectAndGiveUp)
359             return false;
360         }
361       }
362 
363       break;
364     }
365     case GIM_CheckCxxInsnPredicate: {
366       int64_t InsnID = MatchTable[CurrentIdx++];
367       int64_t Predicate = MatchTable[CurrentIdx++];
368       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
369                       dbgs()
370                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
371                           << InsnID << "], Predicate=" << Predicate << ")\n");
372       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
373       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
374 
375       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID],
376                               State.RecordedOperands))
377         if (handleReject() == RejectAndGiveUp)
378           return false;
379       break;
380     }
381     case GIM_CheckAtomicOrdering: {
382       int64_t InsnID = MatchTable[CurrentIdx++];
383       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
384       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
385                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
386                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
387       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
388       if (!State.MIs[InsnID]->hasOneMemOperand())
389         if (handleReject() == RejectAndGiveUp)
390           return false;
391 
392       for (const auto &MMO : State.MIs[InsnID]->memoperands())
393         if (MMO->getOrdering() != Ordering)
394           if (handleReject() == RejectAndGiveUp)
395             return false;
396       break;
397     }
398     case GIM_CheckAtomicOrderingOrStrongerThan: {
399       int64_t InsnID = MatchTable[CurrentIdx++];
400       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
401       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
402                       dbgs() << CurrentIdx
403                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
404                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
405       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
406       if (!State.MIs[InsnID]->hasOneMemOperand())
407         if (handleReject() == RejectAndGiveUp)
408           return false;
409 
410       for (const auto &MMO : State.MIs[InsnID]->memoperands())
411         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
412           if (handleReject() == RejectAndGiveUp)
413             return false;
414       break;
415     }
416     case GIM_CheckAtomicOrderingWeakerThan: {
417       int64_t InsnID = MatchTable[CurrentIdx++];
418       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
419       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
420                       dbgs() << CurrentIdx
421                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
422                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
423       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
424       if (!State.MIs[InsnID]->hasOneMemOperand())
425         if (handleReject() == RejectAndGiveUp)
426           return false;
427 
428       for (const auto &MMO : State.MIs[InsnID]->memoperands())
429         if (!isStrongerThan(Ordering, MMO->getOrdering()))
430           if (handleReject() == RejectAndGiveUp)
431             return false;
432       break;
433     }
434     case GIM_CheckMemoryAddressSpace: {
435       int64_t InsnID = MatchTable[CurrentIdx++];
436       int64_t MMOIdx = MatchTable[CurrentIdx++];
437       // This accepts a list of possible address spaces.
438       const int NumAddrSpace = MatchTable[CurrentIdx++];
439 
440       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
441         if (handleReject() == RejectAndGiveUp)
442           return false;
443         break;
444       }
445 
446       // Need to still jump to the end of the list of address spaces if we find
447       // a match earlier.
448       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
449 
450       const MachineMemOperand *MMO
451         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
452       const unsigned MMOAddrSpace = MMO->getAddrSpace();
453 
454       bool Success = false;
455       for (int I = 0; I != NumAddrSpace; ++I) {
456         unsigned AddrSpace = MatchTable[CurrentIdx++];
457         DEBUG_WITH_TYPE(
458           TgtInstructionSelector::getName(),
459           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
460                  << AddrSpace << '\n');
461 
462         if (AddrSpace == MMOAddrSpace) {
463           Success = true;
464           break;
465         }
466       }
467 
468       CurrentIdx = LastIdx;
469       if (!Success && handleReject() == RejectAndGiveUp)
470         return false;
471       break;
472     }
473     case GIM_CheckMemoryAlignment: {
474       int64_t InsnID = MatchTable[CurrentIdx++];
475       int64_t MMOIdx = MatchTable[CurrentIdx++];
476       unsigned MinAlign = MatchTable[CurrentIdx++];
477 
478       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
479 
480       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
481         if (handleReject() == RejectAndGiveUp)
482           return false;
483         break;
484       }
485 
486       MachineMemOperand *MMO
487         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
488       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
489                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
490                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
491                       << ")->getAlignment() >= " << MinAlign << ")\n");
492       if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
493         return false;
494 
495       break;
496     }
497     case GIM_CheckMemorySizeEqualTo: {
498       int64_t InsnID = MatchTable[CurrentIdx++];
499       int64_t MMOIdx = MatchTable[CurrentIdx++];
500       uint64_t Size = MatchTable[CurrentIdx++];
501 
502       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
503                       dbgs() << CurrentIdx
504                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
505                              << "]->memoperands() + " << MMOIdx
506                              << ", Size=" << Size << ")\n");
507       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
508 
509       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
510         if (handleReject() == RejectAndGiveUp)
511           return false;
512         break;
513       }
514 
515       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
516 
517       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
518                       dbgs() << MMO->getSize() << " bytes vs " << Size
519                              << " bytes\n");
520       if (MMO->getSize() != Size)
521         if (handleReject() == RejectAndGiveUp)
522           return false;
523 
524       break;
525     }
526     case GIM_CheckMemorySizeEqualToLLT:
527     case GIM_CheckMemorySizeLessThanLLT:
528     case GIM_CheckMemorySizeGreaterThanLLT: {
529       int64_t InsnID = MatchTable[CurrentIdx++];
530       int64_t MMOIdx = MatchTable[CurrentIdx++];
531       int64_t OpIdx = MatchTable[CurrentIdx++];
532 
533       DEBUG_WITH_TYPE(
534           TgtInstructionSelector::getName(),
535           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
536                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
537                          ? "EqualTo"
538                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
539                                ? "GreaterThan"
540                                : "LessThan")
541                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
542                  << ", OpIdx=" << OpIdx << ")\n");
543       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
544 
545       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
546       if (!MO.isReg()) {
547         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
548                         dbgs() << CurrentIdx << ": Not a register\n");
549         if (handleReject() == RejectAndGiveUp)
550           return false;
551         break;
552       }
553 
554       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
555         if (handleReject() == RejectAndGiveUp)
556           return false;
557         break;
558       }
559 
560       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
561 
562       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
563       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
564           MMO->getSizeInBits() != Size) {
565         if (handleReject() == RejectAndGiveUp)
566           return false;
567       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
568                  MMO->getSizeInBits() >= Size) {
569         if (handleReject() == RejectAndGiveUp)
570           return false;
571       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
572                  MMO->getSizeInBits() <= Size)
573         if (handleReject() == RejectAndGiveUp)
574           return false;
575 
576       break;
577     }
578     case GIM_CheckType: {
579       int64_t InsnID = MatchTable[CurrentIdx++];
580       int64_t OpIdx = MatchTable[CurrentIdx++];
581       int64_t TypeID = MatchTable[CurrentIdx++];
582       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
583                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
584                              << "]->getOperand(" << OpIdx
585                              << "), TypeID=" << TypeID << ")\n");
586       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
587       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
588       if (!MO.isReg() ||
589           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
590         if (handleReject() == RejectAndGiveUp)
591           return false;
592       }
593       break;
594     }
595     case GIM_CheckPointerToAny: {
596       int64_t InsnID = MatchTable[CurrentIdx++];
597       int64_t OpIdx = MatchTable[CurrentIdx++];
598       int64_t SizeInBits = MatchTable[CurrentIdx++];
599 
600       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
601                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
602                              << InsnID << "]->getOperand(" << OpIdx
603                              << "), SizeInBits=" << SizeInBits << ")\n");
604       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
605       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
606       const LLT Ty = MRI.getType(MO.getReg());
607 
608       // iPTR must be looked up in the target.
609       if (SizeInBits == 0) {
610         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
611         const unsigned AddrSpace = Ty.getAddressSpace();
612         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
613       }
614 
615       assert(SizeInBits != 0 && "Pointer size must be known");
616 
617       if (MO.isReg()) {
618         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
619           if (handleReject() == RejectAndGiveUp)
620             return false;
621       } else if (handleReject() == RejectAndGiveUp)
622         return false;
623 
624       break;
625     }
626     case GIM_RecordNamedOperand: {
627       int64_t InsnID = MatchTable[CurrentIdx++];
628       int64_t OpIdx = MatchTable[CurrentIdx++];
629       uint64_t StoreIdx = MatchTable[CurrentIdx++];
630 
631       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
632                       dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
633                              << InsnID << "]->getOperand(" << OpIdx
634                              << "), StoreIdx=" << StoreIdx << ")\n");
635       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
636       assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
637       State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
638       break;
639     }
640     case GIM_CheckRegBankForClass: {
641       int64_t InsnID = MatchTable[CurrentIdx++];
642       int64_t OpIdx = MatchTable[CurrentIdx++];
643       int64_t RCEnum = MatchTable[CurrentIdx++];
644       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
645                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
646                              << InsnID << "]->getOperand(" << OpIdx
647                              << "), RCEnum=" << RCEnum << ")\n");
648       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
650       if (!MO.isReg() ||
651           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
652                                       MRI.getType(MO.getReg())) !=
653               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
654         if (handleReject() == RejectAndGiveUp)
655           return false;
656       }
657       break;
658     }
659 
660     case GIM_CheckComplexPattern: {
661       int64_t InsnID = MatchTable[CurrentIdx++];
662       int64_t OpIdx = MatchTable[CurrentIdx++];
663       int64_t RendererID = MatchTable[CurrentIdx++];
664       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
665       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
666                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
667                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
668                              << "]->getOperand(" << OpIdx
669                              << "), ComplexPredicateID=" << ComplexPredicateID
670                              << ")\n");
671       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
672       // FIXME: Use std::invoke() when it's available.
673       ComplexRendererFns Renderer =
674           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
675               State.MIs[InsnID]->getOperand(OpIdx));
676       if (Renderer.hasValue())
677         State.Renderers[RendererID] = Renderer.getValue();
678       else
679         if (handleReject() == RejectAndGiveUp)
680           return false;
681       break;
682     }
683 
684     case GIM_CheckConstantInt: {
685       int64_t InsnID = MatchTable[CurrentIdx++];
686       int64_t OpIdx = MatchTable[CurrentIdx++];
687       int64_t Value = MatchTable[CurrentIdx++];
688       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
689                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
690                              << InsnID << "]->getOperand(" << OpIdx
691                              << "), Value=" << Value << ")\n");
692       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
693       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
694       if (MO.isReg()) {
695         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
696         LLT Ty = MRI.getType(MO.getReg());
697         Value = SignExtend64(Value, Ty.getSizeInBits());
698 
699         if (!isOperandImmEqual(MO, Value, MRI)) {
700           if (handleReject() == RejectAndGiveUp)
701             return false;
702         }
703       } else if (handleReject() == RejectAndGiveUp)
704         return false;
705 
706       break;
707     }
708 
709     case GIM_CheckLiteralInt: {
710       int64_t InsnID = MatchTable[CurrentIdx++];
711       int64_t OpIdx = MatchTable[CurrentIdx++];
712       int64_t Value = MatchTable[CurrentIdx++];
713       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
714                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
715                              << InsnID << "]->getOperand(" << OpIdx
716                              << "), Value=" << Value << ")\n");
717       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
718       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
719       if (MO.isImm() && MO.getImm() == Value)
720         break;
721 
722       if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
723         break;
724 
725       if (handleReject() == RejectAndGiveUp)
726         return false;
727 
728       break;
729     }
730 
731     case GIM_CheckIntrinsicID: {
732       int64_t InsnID = MatchTable[CurrentIdx++];
733       int64_t OpIdx = MatchTable[CurrentIdx++];
734       int64_t Value = MatchTable[CurrentIdx++];
735       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
737                              << InsnID << "]->getOperand(" << OpIdx
738                              << "), Value=" << Value << ")\n");
739       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
740       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
741       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
742         if (handleReject() == RejectAndGiveUp)
743           return false;
744       break;
745     }
746     case GIM_CheckCmpPredicate: {
747       int64_t InsnID = MatchTable[CurrentIdx++];
748       int64_t OpIdx = MatchTable[CurrentIdx++];
749       int64_t Value = MatchTable[CurrentIdx++];
750       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
751                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
752                              << InsnID << "]->getOperand(" << OpIdx
753                              << "), Value=" << Value << ")\n");
754       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
755       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
756       if (!MO.isPredicate() || MO.getPredicate() != Value)
757         if (handleReject() == RejectAndGiveUp)
758           return false;
759       break;
760     }
761     case GIM_CheckIsMBB: {
762       int64_t InsnID = MatchTable[CurrentIdx++];
763       int64_t OpIdx = MatchTable[CurrentIdx++];
764       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
765                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
766                              << "]->getOperand(" << OpIdx << "))\n");
767       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
768       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
769         if (handleReject() == RejectAndGiveUp)
770           return false;
771       }
772       break;
773     }
774     case GIM_CheckIsImm: {
775       int64_t InsnID = MatchTable[CurrentIdx++];
776       int64_t OpIdx = MatchTable[CurrentIdx++];
777       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
778                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
779                              << "]->getOperand(" << OpIdx << "))\n");
780       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
781       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
782         if (handleReject() == RejectAndGiveUp)
783           return false;
784       }
785       break;
786     }
787     case GIM_CheckIsSafeToFold: {
788       int64_t InsnID = MatchTable[CurrentIdx++];
789       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
790                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
791                              << InsnID << "])\n");
792       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
793       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
794         if (handleReject() == RejectAndGiveUp)
795           return false;
796       }
797       break;
798     }
799     case GIM_CheckIsSameOperand: {
800       int64_t InsnID = MatchTable[CurrentIdx++];
801       int64_t OpIdx = MatchTable[CurrentIdx++];
802       int64_t OtherInsnID = MatchTable[CurrentIdx++];
803       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
804       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
806                              << InsnID << "][" << OpIdx << "], MIs["
807                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
808       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
809       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
810       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
811               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
812         if (handleReject() == RejectAndGiveUp)
813           return false;
814       }
815       break;
816     }
817     case GIM_Reject:
818       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
819                       dbgs() << CurrentIdx << ": GIM_Reject\n");
820       if (handleReject() == RejectAndGiveUp)
821         return false;
822       break;
823 
824     case GIR_MutateOpcode: {
825       int64_t OldInsnID = MatchTable[CurrentIdx++];
826       uint64_t NewInsnID = MatchTable[CurrentIdx++];
827       int64_t NewOpcode = MatchTable[CurrentIdx++];
828       if (NewInsnID >= OutMIs.size())
829         OutMIs.resize(NewInsnID + 1);
830 
831       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
832                                               State.MIs[OldInsnID]);
833       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
834       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
835                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
836                              << NewInsnID << "], MIs[" << OldInsnID << "], "
837                              << NewOpcode << ")\n");
838       break;
839     }
840 
841     case GIR_BuildMI: {
842       uint64_t NewInsnID = MatchTable[CurrentIdx++];
843       int64_t Opcode = MatchTable[CurrentIdx++];
844       if (NewInsnID >= OutMIs.size())
845         OutMIs.resize(NewInsnID + 1);
846 
847       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
848                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
849       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
850                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
851                              << NewInsnID << "], " << Opcode << ")\n");
852       break;
853     }
854 
855     case GIR_Copy: {
856       int64_t NewInsnID = MatchTable[CurrentIdx++];
857       int64_t OldInsnID = MatchTable[CurrentIdx++];
858       int64_t OpIdx = MatchTable[CurrentIdx++];
859       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
860       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
861       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
862                       dbgs()
863                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
864                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
865       break;
866     }
867 
868     case GIR_CopyOrAddZeroReg: {
869       int64_t NewInsnID = MatchTable[CurrentIdx++];
870       int64_t OldInsnID = MatchTable[CurrentIdx++];
871       int64_t OpIdx = MatchTable[CurrentIdx++];
872       int64_t ZeroReg = MatchTable[CurrentIdx++];
873       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
874       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
875       if (isOperandImmEqual(MO, 0, MRI))
876         OutMIs[NewInsnID].addReg(ZeroReg);
877       else
878         OutMIs[NewInsnID].add(MO);
879       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
881                              << NewInsnID << "], MIs[" << OldInsnID << "], "
882                              << OpIdx << ", " << ZeroReg << ")\n");
883       break;
884     }
885 
886     case GIR_CopySubReg: {
887       int64_t NewInsnID = MatchTable[CurrentIdx++];
888       int64_t OldInsnID = MatchTable[CurrentIdx++];
889       int64_t OpIdx = MatchTable[CurrentIdx++];
890       int64_t SubRegIdx = MatchTable[CurrentIdx++];
891       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
892       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
893                                0, SubRegIdx);
894       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
895                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
896                              << NewInsnID << "], MIs[" << OldInsnID << "], "
897                              << OpIdx << ", " << SubRegIdx << ")\n");
898       break;
899     }
900 
901     case GIR_AddImplicitDef: {
902       int64_t InsnID = MatchTable[CurrentIdx++];
903       int64_t RegNum = MatchTable[CurrentIdx++];
904       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
906       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
907                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
908                              << InsnID << "], " << RegNum << ")\n");
909       break;
910     }
911 
912     case GIR_AddImplicitUse: {
913       int64_t InsnID = MatchTable[CurrentIdx++];
914       int64_t RegNum = MatchTable[CurrentIdx++];
915       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
916       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
917       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
918                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
919                              << InsnID << "], " << RegNum << ")\n");
920       break;
921     }
922 
923     case GIR_AddRegister: {
924       int64_t InsnID = MatchTable[CurrentIdx++];
925       int64_t RegNum = MatchTable[CurrentIdx++];
926       uint64_t RegFlags = MatchTable[CurrentIdx++];
927       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
928       OutMIs[InsnID].addReg(RegNum, RegFlags);
929       DEBUG_WITH_TYPE(
930         TgtInstructionSelector::getName(),
931         dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
932         << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
933       break;
934     }
935 
936     case GIR_AddTempRegister:
937     case GIR_AddTempSubRegister: {
938       int64_t InsnID = MatchTable[CurrentIdx++];
939       int64_t TempRegID = MatchTable[CurrentIdx++];
940       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
941       unsigned SubReg = 0;
942       if (MatcherOpcode == GIR_AddTempSubRegister)
943         SubReg = MatchTable[CurrentIdx++];
944 
945       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
946 
947       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
948       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
949                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
950                              << InsnID << "], TempRegisters[" << TempRegID
951                              << "]";
952                       if (SubReg)
953                         dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
954                       dbgs() << ", " << TempRegFlags << ")\n");
955       break;
956     }
957 
958     case GIR_AddImm: {
959       int64_t InsnID = MatchTable[CurrentIdx++];
960       int64_t Imm = MatchTable[CurrentIdx++];
961       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
962       OutMIs[InsnID].addImm(Imm);
963       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
964                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
965                              << "], " << Imm << ")\n");
966       break;
967     }
968 
969     case GIR_ComplexRenderer: {
970       int64_t InsnID = MatchTable[CurrentIdx++];
971       int64_t RendererID = MatchTable[CurrentIdx++];
972       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
973       for (const auto &RenderOpFn : State.Renderers[RendererID])
974         RenderOpFn(OutMIs[InsnID]);
975       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
976                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
977                              << InsnID << "], " << RendererID << ")\n");
978       break;
979     }
980     case GIR_ComplexSubOperandRenderer: {
981       int64_t InsnID = MatchTable[CurrentIdx++];
982       int64_t RendererID = MatchTable[CurrentIdx++];
983       int64_t RenderOpID = MatchTable[CurrentIdx++];
984       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
985       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
986       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
987                       dbgs() << CurrentIdx
988                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
989                              << InsnID << "], " << RendererID << ", "
990                              << RenderOpID << ")\n");
991       break;
992     }
993 
994     case GIR_CopyConstantAsSImm: {
995       int64_t NewInsnID = MatchTable[CurrentIdx++];
996       int64_t OldInsnID = MatchTable[CurrentIdx++];
997       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
998       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
999       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1000         OutMIs[NewInsnID].addImm(
1001             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1002       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1003         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1004       else
1005         llvm_unreachable("Expected Imm or CImm operand");
1006       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1007                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1008                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1009       break;
1010     }
1011 
1012     // TODO: Needs a test case once we have a pattern that uses this.
1013     case GIR_CopyFConstantAsFPImm: {
1014       int64_t NewInsnID = MatchTable[CurrentIdx++];
1015       int64_t OldInsnID = MatchTable[CurrentIdx++];
1016       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1017       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
1018       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1019         OutMIs[NewInsnID].addFPImm(
1020             State.MIs[OldInsnID]->getOperand(1).getFPImm());
1021       else
1022         llvm_unreachable("Expected FPImm operand");
1023       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1024                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1025                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1026       break;
1027     }
1028 
1029     case GIR_CustomRenderer: {
1030       int64_t InsnID = MatchTable[CurrentIdx++];
1031       int64_t OldInsnID = MatchTable[CurrentIdx++];
1032       int64_t RendererFnID = MatchTable[CurrentIdx++];
1033       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1034       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1035                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1036                              << InsnID << "], MIs[" << OldInsnID << "], "
1037                              << RendererFnID << ")\n");
1038       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
1039         OutMIs[InsnID], *State.MIs[OldInsnID],
1040         -1); // Not a source operand of the old instruction.
1041       break;
1042     }
1043     case GIR_CustomOperandRenderer: {
1044       int64_t InsnID = MatchTable[CurrentIdx++];
1045       int64_t OldInsnID = MatchTable[CurrentIdx++];
1046       int64_t OpIdx = MatchTable[CurrentIdx++];
1047       int64_t RendererFnID = MatchTable[CurrentIdx++];
1048       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1049 
1050       DEBUG_WITH_TYPE(
1051         TgtInstructionSelector::getName(),
1052         dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
1053                << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
1054                << OpIdx << "), "
1055         << RendererFnID << ")\n");
1056       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
1057                                                      *State.MIs[OldInsnID],
1058                                                      OpIdx);
1059       break;
1060     }
1061     case GIR_ConstrainOperandRC: {
1062       int64_t InsnID = MatchTable[CurrentIdx++];
1063       int64_t OpIdx = MatchTable[CurrentIdx++];
1064       int64_t RCEnum = MatchTable[CurrentIdx++];
1065       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1066       MachineInstr &I = *OutMIs[InsnID].getInstr();
1067       MachineFunction &MF = *I.getParent()->getParent();
1068       MachineRegisterInfo &MRI = MF.getRegInfo();
1069       const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1070       MachineOperand &MO = I.getOperand(OpIdx);
1071       constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1072       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1073                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1074                              << InsnID << "], " << OpIdx << ", " << RCEnum
1075                              << ")\n");
1076       break;
1077     }
1078 
1079     case GIR_ConstrainSelectedInstOperands: {
1080       int64_t InsnID = MatchTable[CurrentIdx++];
1081       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1082       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1083                                        RBI);
1084       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1085                       dbgs() << CurrentIdx
1086                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1087                              << InsnID << "])\n");
1088       break;
1089     }
1090 
1091     case GIR_MergeMemOperands: {
1092       int64_t InsnID = MatchTable[CurrentIdx++];
1093       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1094 
1095       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1096                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1097                              << InsnID << "]");
1098       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1099       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1100              GIU_MergeMemOperands_EndOfList) {
1101         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1102                         dbgs() << ", MIs[" << MergeInsnID << "]");
1103         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1104           OutMIs[InsnID].addMemOperand(MMO);
1105       }
1106       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
1107       break;
1108     }
1109 
1110     case GIR_EraseFromParent: {
1111       int64_t InsnID = MatchTable[CurrentIdx++];
1112       assert(State.MIs[InsnID] &&
1113              "Attempted to erase an undefined instruction");
1114       State.MIs[InsnID]->eraseFromParent();
1115       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1116                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1117                              << InsnID << "])\n");
1118       break;
1119     }
1120 
1121     case GIR_MakeTempReg: {
1122       int64_t TempRegID = MatchTable[CurrentIdx++];
1123       int64_t TypeID = MatchTable[CurrentIdx++];
1124 
1125       State.TempRegisters[TempRegID] =
1126           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1127       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1128                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1129                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1130       break;
1131     }
1132 
1133     case GIR_Coverage: {
1134       int64_t RuleID = MatchTable[CurrentIdx++];
1135       CoverageInfo.setCovered(RuleID);
1136 
1137       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1138                       dbgs()
1139                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1140       break;
1141     }
1142 
1143     case GIR_Done:
1144       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1145                       dbgs() << CurrentIdx << ": GIR_Done\n");
1146       propagateFlags(OutMIs);
1147       return true;
1148 
1149     default:
1150       llvm_unreachable("Unexpected command");
1151     }
1152   }
1153 }
1154 
1155 } // end namespace llvm
1156 
1157 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
1158