1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file declares the API for the instruction selector.
10 /// This class is responsible for selecting machine instructions.
11 /// It's implemented by the target. It's used by the InstructionSelect pass.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20 #include "llvm/CodeGen/GlobalISel/Utils.h"
21 #include "llvm/CodeGen/MachineInstrBuilder.h"
22 #include "llvm/CodeGen/MachineOperand.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/CodeGen/RegisterBankInfo.h"
25 #include "llvm/CodeGen/TargetInstrInfo.h"
26 #include "llvm/CodeGen/TargetOpcodes.h"
27 #include "llvm/CodeGen/TargetRegisterInfo.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/Support/CodeGenCoverage.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/ErrorHandling.h"
33 #include "llvm/Support/raw_ostream.h"
34 #include <cassert>
35 #include <cstddef>
36 #include <cstdint>
37
38 namespace llvm {
39
40 /// GlobalISel PatFrag Predicates
41 enum {
42 GIPFP_I64_Invalid = 0,
43 GIPFP_APInt_Invalid = 0,
44 GIPFP_APFloat_Invalid = 0,
45 GIPFP_MI_Invalid = 0,
46 };
47
48 template <class TgtInstructionSelector, class PredicateBitset,
49 class ComplexMatcherMemFn, class CustomRendererFn>
executeMatchTable(TgtInstructionSelector & ISel,NewMIVector & OutMIs,MatcherState & State,const ISelInfoTy<PredicateBitset,ComplexMatcherMemFn,CustomRendererFn> & ISelInfo,const int64_t * MatchTable,const TargetInstrInfo & TII,MachineRegisterInfo & MRI,const TargetRegisterInfo & TRI,const RegisterBankInfo & RBI,const PredicateBitset & AvailableFeatures,CodeGenCoverage & CoverageInfo)50 bool InstructionSelector::executeMatchTable(
51 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
52 const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
53 &ISelInfo,
54 const int64_t *MatchTable, const TargetInstrInfo &TII,
55 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
56 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
57 CodeGenCoverage &CoverageInfo) const {
58
59 uint64_t CurrentIdx = 0;
60 SmallVector<uint64_t, 4> OnFailResumeAt;
61
62 // Bypass the flag check on the instruction, and only look at the MCInstrDesc.
63 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();
64
65 const uint16_t Flags = State.MIs[0]->getFlags();
66
67 enum RejectAction { RejectAndGiveUp, RejectAndResume };
68 auto handleReject = [&]() -> RejectAction {
69 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
70 dbgs() << CurrentIdx << ": Rejected\n");
71 if (OnFailResumeAt.empty())
72 return RejectAndGiveUp;
73 CurrentIdx = OnFailResumeAt.pop_back_val();
74 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
75 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
76 << OnFailResumeAt.size() << " try-blocks remain)\n");
77 return RejectAndResume;
78 };
79
80 auto propagateFlags = [=](NewMIVector &OutMIs) {
81 for (auto MIB : OutMIs) {
82 // Set the NoFPExcept flag when no original matched instruction could
83 // raise an FP exception, but the new instruction potentially might.
84 uint16_t MIBFlags = Flags;
85 if (NoFPException && MIB->mayRaiseFPException())
86 MIBFlags |= MachineInstr::NoFPExcept;
87 MIB.setMIFlags(MIBFlags);
88 }
89
90 return true;
91 };
92
93 while (true) {
94 assert(CurrentIdx != ~0u && "Invalid MatchTable index");
95 int64_t MatcherOpcode = MatchTable[CurrentIdx++];
96 switch (MatcherOpcode) {
97 case GIM_Try: {
98 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
99 dbgs() << CurrentIdx << ": Begin try-block\n");
100 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
101 break;
102 }
103
104 case GIM_RecordInsn: {
105 int64_t NewInsnID = MatchTable[CurrentIdx++];
106 int64_t InsnID = MatchTable[CurrentIdx++];
107 int64_t OpIdx = MatchTable[CurrentIdx++];
108
109 // As an optimisation we require that MIs[0] is always the root. Refuse
110 // any attempt to modify it.
111 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
112
113 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
114 if (!MO.isReg()) {
115 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
116 dbgs() << CurrentIdx << ": Not a register\n");
117 if (handleReject() == RejectAndGiveUp)
118 return false;
119 break;
120 }
121 if (MO.getReg().isPhysical()) {
122 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
123 dbgs() << CurrentIdx << ": Is a physical register\n");
124 if (handleReject() == RejectAndGiveUp)
125 return false;
126 break;
127 }
128
129 MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
130 if ((size_t)NewInsnID < State.MIs.size())
131 State.MIs[NewInsnID] = NewMI;
132 else {
133 assert((size_t)NewInsnID == State.MIs.size() &&
134 "Expected to store MIs in order");
135 State.MIs.push_back(NewMI);
136 }
137 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
138 dbgs() << CurrentIdx << ": MIs[" << NewInsnID
139 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
140 << ")\n");
141 break;
142 }
143
144 case GIM_CheckFeatures: {
145 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
146 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147 dbgs() << CurrentIdx
148 << ": GIM_CheckFeatures(ExpectedBitsetID="
149 << ExpectedBitsetID << ")\n");
150 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
151 ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
152 if (handleReject() == RejectAndGiveUp)
153 return false;
154 }
155 break;
156 }
157
158 case GIM_CheckOpcode:
159 case GIM_CheckOpcodeIsEither: {
160 int64_t InsnID = MatchTable[CurrentIdx++];
161 int64_t Expected0 = MatchTable[CurrentIdx++];
162 int64_t Expected1 = -1;
163 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
164 Expected1 = MatchTable[CurrentIdx++];
165
166 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
167 unsigned Opcode = State.MIs[InsnID]->getOpcode();
168
169 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
170 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
171 << "], ExpectedOpcode=" << Expected0;
172 if (MatcherOpcode == GIM_CheckOpcodeIsEither)
173 dbgs() << " || " << Expected1;
174 dbgs() << ") // Got=" << Opcode << "\n";
175 );
176
177 if (Opcode != Expected0 && Opcode != Expected1) {
178 if (handleReject() == RejectAndGiveUp)
179 return false;
180 }
181 break;
182 }
183 case GIM_SwitchOpcode: {
184 int64_t InsnID = MatchTable[CurrentIdx++];
185 int64_t LowerBound = MatchTable[CurrentIdx++];
186 int64_t UpperBound = MatchTable[CurrentIdx++];
187 int64_t Default = MatchTable[CurrentIdx++];
188
189 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
190 const int64_t Opcode = State.MIs[InsnID]->getOpcode();
191
192 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
193 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
194 << LowerBound << ", " << UpperBound << "), Default=" << Default
195 << ", JumpTable...) // Got=" << Opcode << "\n";
196 });
197 if (Opcode < LowerBound || UpperBound <= Opcode) {
198 CurrentIdx = Default;
199 break;
200 }
201 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
202 if (!CurrentIdx) {
203 CurrentIdx = Default;
204 break;
205 }
206 OnFailResumeAt.push_back(Default);
207 break;
208 }
209
210 case GIM_SwitchType: {
211 int64_t InsnID = MatchTable[CurrentIdx++];
212 int64_t OpIdx = MatchTable[CurrentIdx++];
213 int64_t LowerBound = MatchTable[CurrentIdx++];
214 int64_t UpperBound = MatchTable[CurrentIdx++];
215 int64_t Default = MatchTable[CurrentIdx++];
216
217 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
218 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
219
220 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
221 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
222 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
223 << UpperBound << "), Default=" << Default
224 << ", JumpTable...) // Got=";
225 if (!MO.isReg())
226 dbgs() << "Not a VReg\n";
227 else
228 dbgs() << MRI.getType(MO.getReg()) << "\n";
229 });
230 if (!MO.isReg()) {
231 CurrentIdx = Default;
232 break;
233 }
234 const LLT Ty = MRI.getType(MO.getReg());
235 const auto TyI = ISelInfo.TypeIDMap.find(Ty);
236 if (TyI == ISelInfo.TypeIDMap.end()) {
237 CurrentIdx = Default;
238 break;
239 }
240 const int64_t TypeID = TyI->second;
241 if (TypeID < LowerBound || UpperBound <= TypeID) {
242 CurrentIdx = Default;
243 break;
244 }
245 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
246 if (!CurrentIdx) {
247 CurrentIdx = Default;
248 break;
249 }
250 OnFailResumeAt.push_back(Default);
251 break;
252 }
253
254 case GIM_CheckNumOperands: {
255 int64_t InsnID = MatchTable[CurrentIdx++];
256 int64_t Expected = MatchTable[CurrentIdx++];
257 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
258 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
259 << InsnID << "], Expected=" << Expected << ")\n");
260 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
261 if (State.MIs[InsnID]->getNumOperands() != Expected) {
262 if (handleReject() == RejectAndGiveUp)
263 return false;
264 }
265 break;
266 }
267 case GIM_CheckI64ImmPredicate:
268 case GIM_CheckImmOperandPredicate: {
269 int64_t InsnID = MatchTable[CurrentIdx++];
270 int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate
271 ? MatchTable[CurrentIdx++]
272 : 1;
273 int64_t Predicate = MatchTable[CurrentIdx++];
274 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
275 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
276 << InsnID << "]->getOperand(" << OpIdx
277 << "), Predicate=" << Predicate << ")\n");
278 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
279 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||
280 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&
281 "Expected immediate operand");
282 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
283 int64_t Value = 0;
284 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())
285 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();
286 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())
287 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();
288 else
289 llvm_unreachable("Expected Imm or CImm operand");
290
291 if (!testImmPredicate_I64(Predicate, Value))
292 if (handleReject() == RejectAndGiveUp)
293 return false;
294 break;
295 }
296 case GIM_CheckAPIntImmPredicate: {
297 int64_t InsnID = MatchTable[CurrentIdx++];
298 int64_t Predicate = MatchTable[CurrentIdx++];
299 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
300 dbgs()
301 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
302 << InsnID << "], Predicate=" << Predicate << ")\n");
303 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
304 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
305 "Expected G_CONSTANT");
306 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
307 APInt Value;
308 if (State.MIs[InsnID]->getOperand(1).isCImm())
309 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
310 else
311 llvm_unreachable("Expected Imm or CImm operand");
312
313 if (!testImmPredicate_APInt(Predicate, Value))
314 if (handleReject() == RejectAndGiveUp)
315 return false;
316 break;
317 }
318 case GIM_CheckAPFloatImmPredicate: {
319 int64_t InsnID = MatchTable[CurrentIdx++];
320 int64_t Predicate = MatchTable[CurrentIdx++];
321 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
322 dbgs()
323 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
324 << InsnID << "], Predicate=" << Predicate << ")\n");
325 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
326 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
327 "Expected G_FCONSTANT");
328 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
329 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
330 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
331
332 if (!testImmPredicate_APFloat(Predicate, Value))
333 if (handleReject() == RejectAndGiveUp)
334 return false;
335 break;
336 }
337 case GIM_CheckIsBuildVectorAllOnes:
338 case GIM_CheckIsBuildVectorAllZeros: {
339 int64_t InsnID = MatchTable[CurrentIdx++];
340
341 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342 dbgs() << CurrentIdx
343 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["
344 << InsnID << "])\n");
345 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
347 const MachineInstr *MI = State.MIs[InsnID];
348 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
349 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&
350 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");
351
352 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) {
353 if (!isBuildVectorAllOnes(*MI, MRI)) {
354 if (handleReject() == RejectAndGiveUp)
355 return false;
356 }
357 } else {
358 if (!isBuildVectorAllZeros(*MI, MRI)) {
359 if (handleReject() == RejectAndGiveUp)
360 return false;
361 }
362 }
363
364 break;
365 }
366 case GIM_CheckCxxInsnPredicate: {
367 int64_t InsnID = MatchTable[CurrentIdx++];
368 int64_t Predicate = MatchTable[CurrentIdx++];
369 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
370 dbgs()
371 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
372 << InsnID << "], Predicate=" << Predicate << ")\n");
373 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
374 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
375
376 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID],
377 State.RecordedOperands))
378 if (handleReject() == RejectAndGiveUp)
379 return false;
380 break;
381 }
382 case GIM_CheckHasNoUse: {
383 int64_t InsnID = MatchTable[CurrentIdx++];
384
385 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
386 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
387 << InsnID << "]\n");
388
389 const MachineInstr *MI = State.MIs[InsnID];
390 assert(MI && "Used insn before defined");
391 assert(MI->getNumDefs() > 0 && "No defs");
392 const Register Res = MI->getOperand(0).getReg();
393
394 if (!MRI.use_nodbg_empty(Res)) {
395 if (handleReject() == RejectAndGiveUp)
396 return false;
397 }
398
399 break;
400 }
401 case GIM_CheckAtomicOrdering: {
402 int64_t InsnID = MatchTable[CurrentIdx++];
403 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
404 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
405 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
406 << InsnID << "], " << (uint64_t)Ordering << ")\n");
407 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
408 if (!State.MIs[InsnID]->hasOneMemOperand())
409 if (handleReject() == RejectAndGiveUp)
410 return false;
411
412 for (const auto &MMO : State.MIs[InsnID]->memoperands())
413 if (MMO->getMergedOrdering() != Ordering)
414 if (handleReject() == RejectAndGiveUp)
415 return false;
416 break;
417 }
418 case GIM_CheckAtomicOrderingOrStrongerThan: {
419 int64_t InsnID = MatchTable[CurrentIdx++];
420 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
421 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
422 dbgs() << CurrentIdx
423 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
424 << InsnID << "], " << (uint64_t)Ordering << ")\n");
425 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
426 if (!State.MIs[InsnID]->hasOneMemOperand())
427 if (handleReject() == RejectAndGiveUp)
428 return false;
429
430 for (const auto &MMO : State.MIs[InsnID]->memoperands())
431 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering))
432 if (handleReject() == RejectAndGiveUp)
433 return false;
434 break;
435 }
436 case GIM_CheckAtomicOrderingWeakerThan: {
437 int64_t InsnID = MatchTable[CurrentIdx++];
438 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
439 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
440 dbgs() << CurrentIdx
441 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
442 << InsnID << "], " << (uint64_t)Ordering << ")\n");
443 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
444 if (!State.MIs[InsnID]->hasOneMemOperand())
445 if (handleReject() == RejectAndGiveUp)
446 return false;
447
448 for (const auto &MMO : State.MIs[InsnID]->memoperands())
449 if (!isStrongerThan(Ordering, MMO->getMergedOrdering()))
450 if (handleReject() == RejectAndGiveUp)
451 return false;
452 break;
453 }
454 case GIM_CheckMemoryAddressSpace: {
455 int64_t InsnID = MatchTable[CurrentIdx++];
456 int64_t MMOIdx = MatchTable[CurrentIdx++];
457 // This accepts a list of possible address spaces.
458 const int NumAddrSpace = MatchTable[CurrentIdx++];
459
460 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
461 if (handleReject() == RejectAndGiveUp)
462 return false;
463 break;
464 }
465
466 // Need to still jump to the end of the list of address spaces if we find
467 // a match earlier.
468 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
469
470 const MachineMemOperand *MMO
471 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
472 const unsigned MMOAddrSpace = MMO->getAddrSpace();
473
474 bool Success = false;
475 for (int I = 0; I != NumAddrSpace; ++I) {
476 unsigned AddrSpace = MatchTable[CurrentIdx++];
477 DEBUG_WITH_TYPE(
478 TgtInstructionSelector::getName(),
479 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
480 << AddrSpace << '\n');
481
482 if (AddrSpace == MMOAddrSpace) {
483 Success = true;
484 break;
485 }
486 }
487
488 CurrentIdx = LastIdx;
489 if (!Success && handleReject() == RejectAndGiveUp)
490 return false;
491 break;
492 }
493 case GIM_CheckMemoryAlignment: {
494 int64_t InsnID = MatchTable[CurrentIdx++];
495 int64_t MMOIdx = MatchTable[CurrentIdx++];
496 unsigned MinAlign = MatchTable[CurrentIdx++];
497
498 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
499
500 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
501 if (handleReject() == RejectAndGiveUp)
502 return false;
503 break;
504 }
505
506 MachineMemOperand *MMO
507 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
508 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
509 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
510 << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
511 << ")->getAlignment() >= " << MinAlign << ")\n");
512 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)
513 return false;
514
515 break;
516 }
517 case GIM_CheckMemorySizeEqualTo: {
518 int64_t InsnID = MatchTable[CurrentIdx++];
519 int64_t MMOIdx = MatchTable[CurrentIdx++];
520 uint64_t Size = MatchTable[CurrentIdx++];
521
522 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
523 dbgs() << CurrentIdx
524 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
525 << "]->memoperands() + " << MMOIdx
526 << ", Size=" << Size << ")\n");
527 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
528
529 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
530 if (handleReject() == RejectAndGiveUp)
531 return false;
532 break;
533 }
534
535 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
536
537 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
538 dbgs() << MMO->getSize() << " bytes vs " << Size
539 << " bytes\n");
540 if (MMO->getSize() != Size)
541 if (handleReject() == RejectAndGiveUp)
542 return false;
543
544 break;
545 }
546 case GIM_CheckMemorySizeEqualToLLT:
547 case GIM_CheckMemorySizeLessThanLLT:
548 case GIM_CheckMemorySizeGreaterThanLLT: {
549 int64_t InsnID = MatchTable[CurrentIdx++];
550 int64_t MMOIdx = MatchTable[CurrentIdx++];
551 int64_t OpIdx = MatchTable[CurrentIdx++];
552
553 DEBUG_WITH_TYPE(
554 TgtInstructionSelector::getName(),
555 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
556 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
557 ? "EqualTo"
558 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
559 ? "GreaterThan"
560 : "LessThan")
561 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
562 << ", OpIdx=" << OpIdx << ")\n");
563 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
564
565 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
566 if (!MO.isReg()) {
567 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
568 dbgs() << CurrentIdx << ": Not a register\n");
569 if (handleReject() == RejectAndGiveUp)
570 return false;
571 break;
572 }
573
574 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
575 if (handleReject() == RejectAndGiveUp)
576 return false;
577 break;
578 }
579
580 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
581
582 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
583 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
584 MMO->getSizeInBits() != Size) {
585 if (handleReject() == RejectAndGiveUp)
586 return false;
587 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
588 MMO->getSizeInBits() >= Size) {
589 if (handleReject() == RejectAndGiveUp)
590 return false;
591 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
592 MMO->getSizeInBits() <= Size)
593 if (handleReject() == RejectAndGiveUp)
594 return false;
595
596 break;
597 }
598 case GIM_CheckType: {
599 int64_t InsnID = MatchTable[CurrentIdx++];
600 int64_t OpIdx = MatchTable[CurrentIdx++];
601 int64_t TypeID = MatchTable[CurrentIdx++];
602 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
603 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
604 << "]->getOperand(" << OpIdx
605 << "), TypeID=" << TypeID << ")\n");
606 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
607 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
608 if (!MO.isReg() ||
609 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
610 if (handleReject() == RejectAndGiveUp)
611 return false;
612 }
613 break;
614 }
615 case GIM_CheckPointerToAny: {
616 int64_t InsnID = MatchTable[CurrentIdx++];
617 int64_t OpIdx = MatchTable[CurrentIdx++];
618 uint64_t SizeInBits = MatchTable[CurrentIdx++];
619
620 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
621 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
622 << InsnID << "]->getOperand(" << OpIdx
623 << "), SizeInBits=" << SizeInBits << ")\n");
624 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
625 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
626 const LLT Ty = MRI.getType(MO.getReg());
627
628 // iPTR must be looked up in the target.
629 if (SizeInBits == 0) {
630 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
631 const unsigned AddrSpace = Ty.getAddressSpace();
632 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
633 }
634
635 assert(SizeInBits != 0 && "Pointer size must be known");
636
637 if (MO.isReg()) {
638 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
639 if (handleReject() == RejectAndGiveUp)
640 return false;
641 } else if (handleReject() == RejectAndGiveUp)
642 return false;
643
644 break;
645 }
646 case GIM_RecordNamedOperand: {
647 int64_t InsnID = MatchTable[CurrentIdx++];
648 int64_t OpIdx = MatchTable[CurrentIdx++];
649 uint64_t StoreIdx = MatchTable[CurrentIdx++];
650
651 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
652 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
653 << InsnID << "]->getOperand(" << OpIdx
654 << "), StoreIdx=" << StoreIdx << ")\n");
655 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
656 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");
657 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);
658 break;
659 }
660 case GIM_CheckRegBankForClass: {
661 int64_t InsnID = MatchTable[CurrentIdx++];
662 int64_t OpIdx = MatchTable[CurrentIdx++];
663 int64_t RCEnum = MatchTable[CurrentIdx++];
664 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
665 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
666 << InsnID << "]->getOperand(" << OpIdx
667 << "), RCEnum=" << RCEnum << ")\n");
668 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
669 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
670 if (!MO.isReg() ||
671 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum),
672 MRI.getType(MO.getReg())) !=
673 RBI.getRegBank(MO.getReg(), MRI, TRI)) {
674 if (handleReject() == RejectAndGiveUp)
675 return false;
676 }
677 break;
678 }
679
680 case GIM_CheckComplexPattern: {
681 int64_t InsnID = MatchTable[CurrentIdx++];
682 int64_t OpIdx = MatchTable[CurrentIdx++];
683 int64_t RendererID = MatchTable[CurrentIdx++];
684 int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
685 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
686 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
687 << "] = GIM_CheckComplexPattern(MIs[" << InsnID
688 << "]->getOperand(" << OpIdx
689 << "), ComplexPredicateID=" << ComplexPredicateID
690 << ")\n");
691 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
692 // FIXME: Use std::invoke() when it's available.
693 ComplexRendererFns Renderer =
694 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
695 State.MIs[InsnID]->getOperand(OpIdx));
696 if (Renderer)
697 State.Renderers[RendererID] = *Renderer;
698 else
699 if (handleReject() == RejectAndGiveUp)
700 return false;
701 break;
702 }
703
704 case GIM_CheckConstantInt: {
705 int64_t InsnID = MatchTable[CurrentIdx++];
706 int64_t OpIdx = MatchTable[CurrentIdx++];
707 int64_t Value = MatchTable[CurrentIdx++];
708 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
710 << InsnID << "]->getOperand(" << OpIdx
711 << "), Value=" << Value << ")\n");
712 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
713 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
714 if (MO.isReg()) {
715 // isOperandImmEqual() will sign-extend to 64-bits, so should we.
716 LLT Ty = MRI.getType(MO.getReg());
717 Value = SignExtend64(Value, Ty.getSizeInBits());
718
719 if (!isOperandImmEqual(MO, Value, MRI)) {
720 if (handleReject() == RejectAndGiveUp)
721 return false;
722 }
723 } else if (handleReject() == RejectAndGiveUp)
724 return false;
725
726 break;
727 }
728
729 case GIM_CheckLiteralInt: {
730 int64_t InsnID = MatchTable[CurrentIdx++];
731 int64_t OpIdx = MatchTable[CurrentIdx++];
732 int64_t Value = MatchTable[CurrentIdx++];
733 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
734 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
735 << InsnID << "]->getOperand(" << OpIdx
736 << "), Value=" << Value << ")\n");
737 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
738 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
739 if (MO.isImm() && MO.getImm() == Value)
740 break;
741
742 if (MO.isCImm() && MO.getCImm()->equalsInt(Value))
743 break;
744
745 if (handleReject() == RejectAndGiveUp)
746 return false;
747
748 break;
749 }
750
751 case GIM_CheckIntrinsicID: {
752 int64_t InsnID = MatchTable[CurrentIdx++];
753 int64_t OpIdx = MatchTable[CurrentIdx++];
754 int64_t Value = MatchTable[CurrentIdx++];
755 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
756 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
757 << InsnID << "]->getOperand(" << OpIdx
758 << "), Value=" << Value << ")\n");
759 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
760 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
761 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
762 if (handleReject() == RejectAndGiveUp)
763 return false;
764 break;
765 }
766 case GIM_CheckCmpPredicate: {
767 int64_t InsnID = MatchTable[CurrentIdx++];
768 int64_t OpIdx = MatchTable[CurrentIdx++];
769 int64_t Value = MatchTable[CurrentIdx++];
770 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
771 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
772 << InsnID << "]->getOperand(" << OpIdx
773 << "), Value=" << Value << ")\n");
774 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
775 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
776 if (!MO.isPredicate() || MO.getPredicate() != Value)
777 if (handleReject() == RejectAndGiveUp)
778 return false;
779 break;
780 }
781 case GIM_CheckIsMBB: {
782 int64_t InsnID = MatchTable[CurrentIdx++];
783 int64_t OpIdx = MatchTable[CurrentIdx++];
784 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
785 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
786 << "]->getOperand(" << OpIdx << "))\n");
787 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
788 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
789 if (handleReject() == RejectAndGiveUp)
790 return false;
791 }
792 break;
793 }
794 case GIM_CheckIsImm: {
795 int64_t InsnID = MatchTable[CurrentIdx++];
796 int64_t OpIdx = MatchTable[CurrentIdx++];
797 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
799 << "]->getOperand(" << OpIdx << "))\n");
800 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
801 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
802 if (handleReject() == RejectAndGiveUp)
803 return false;
804 }
805 break;
806 }
807 case GIM_CheckIsSafeToFold: {
808 int64_t InsnID = MatchTable[CurrentIdx++];
809 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
811 << InsnID << "])\n");
812 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
813 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
814 if (handleReject() == RejectAndGiveUp)
815 return false;
816 }
817 break;
818 }
819 case GIM_CheckIsSameOperand: {
820 int64_t InsnID = MatchTable[CurrentIdx++];
821 int64_t OpIdx = MatchTable[CurrentIdx++];
822 int64_t OtherInsnID = MatchTable[CurrentIdx++];
823 int64_t OtherOpIdx = MatchTable[CurrentIdx++];
824 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
826 << InsnID << "][" << OpIdx << "], MIs["
827 << OtherInsnID << "][" << OtherOpIdx << "])\n");
828 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
829 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
830 if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
831 State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
832 if (handleReject() == RejectAndGiveUp)
833 return false;
834 }
835 break;
836 }
837 case GIM_Reject:
838 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
839 dbgs() << CurrentIdx << ": GIM_Reject\n");
840 if (handleReject() == RejectAndGiveUp)
841 return false;
842 break;
843
844 case GIR_MutateOpcode: {
845 int64_t OldInsnID = MatchTable[CurrentIdx++];
846 uint64_t NewInsnID = MatchTable[CurrentIdx++];
847 int64_t NewOpcode = MatchTable[CurrentIdx++];
848 if (NewInsnID >= OutMIs.size())
849 OutMIs.resize(NewInsnID + 1);
850
851 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
852 State.MIs[OldInsnID]);
853 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
854 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
855 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
856 << NewInsnID << "], MIs[" << OldInsnID << "], "
857 << NewOpcode << ")\n");
858 break;
859 }
860
861 case GIR_BuildMI: {
862 uint64_t NewInsnID = MatchTable[CurrentIdx++];
863 int64_t Opcode = MatchTable[CurrentIdx++];
864 if (NewInsnID >= OutMIs.size())
865 OutMIs.resize(NewInsnID + 1);
866
867 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
868 MIMetadata(*State.MIs[0]), TII.get(Opcode));
869 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
870 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
871 << NewInsnID << "], " << Opcode << ")\n");
872 break;
873 }
874
875 case GIR_Copy: {
876 int64_t NewInsnID = MatchTable[CurrentIdx++];
877 int64_t OldInsnID = MatchTable[CurrentIdx++];
878 int64_t OpIdx = MatchTable[CurrentIdx++];
879 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
880 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
881 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882 dbgs()
883 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
884 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
885 break;
886 }
887
888 case GIR_CopyOrAddZeroReg: {
889 int64_t NewInsnID = MatchTable[CurrentIdx++];
890 int64_t OldInsnID = MatchTable[CurrentIdx++];
891 int64_t OpIdx = MatchTable[CurrentIdx++];
892 int64_t ZeroReg = MatchTable[CurrentIdx++];
893 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
894 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
895 if (isOperandImmEqual(MO, 0, MRI))
896 OutMIs[NewInsnID].addReg(ZeroReg);
897 else
898 OutMIs[NewInsnID].add(MO);
899 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
900 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
901 << NewInsnID << "], MIs[" << OldInsnID << "], "
902 << OpIdx << ", " << ZeroReg << ")\n");
903 break;
904 }
905
906 case GIR_CopySubReg: {
907 int64_t NewInsnID = MatchTable[CurrentIdx++];
908 int64_t OldInsnID = MatchTable[CurrentIdx++];
909 int64_t OpIdx = MatchTable[CurrentIdx++];
910 int64_t SubRegIdx = MatchTable[CurrentIdx++];
911 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
912 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
913 0, SubRegIdx);
914 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
915 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
916 << NewInsnID << "], MIs[" << OldInsnID << "], "
917 << OpIdx << ", " << SubRegIdx << ")\n");
918 break;
919 }
920
921 case GIR_AddImplicitDef: {
922 int64_t InsnID = MatchTable[CurrentIdx++];
923 int64_t RegNum = MatchTable[CurrentIdx++];
924 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
925 OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
926 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
927 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
928 << InsnID << "], " << RegNum << ")\n");
929 break;
930 }
931
932 case GIR_AddImplicitUse: {
933 int64_t InsnID = MatchTable[CurrentIdx++];
934 int64_t RegNum = MatchTable[CurrentIdx++];
935 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
936 OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
937 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
938 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
939 << InsnID << "], " << RegNum << ")\n");
940 break;
941 }
942
943 case GIR_AddRegister: {
944 int64_t InsnID = MatchTable[CurrentIdx++];
945 int64_t RegNum = MatchTable[CurrentIdx++];
946 uint64_t RegFlags = MatchTable[CurrentIdx++];
947 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
948 OutMIs[InsnID].addReg(RegNum, RegFlags);
949 DEBUG_WITH_TYPE(
950 TgtInstructionSelector::getName(),
951 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
952 << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
953 break;
954 }
955
956 case GIR_AddTempRegister:
957 case GIR_AddTempSubRegister: {
958 int64_t InsnID = MatchTable[CurrentIdx++];
959 int64_t TempRegID = MatchTable[CurrentIdx++];
960 uint64_t TempRegFlags = MatchTable[CurrentIdx++];
961 unsigned SubReg = 0;
962 if (MatcherOpcode == GIR_AddTempSubRegister)
963 SubReg = MatchTable[CurrentIdx++];
964
965 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
966
967 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg);
968 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
969 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
970 << InsnID << "], TempRegisters[" << TempRegID
971 << "]";
972 if (SubReg)
973 dbgs() << '.' << TRI.getSubRegIndexName(SubReg);
974 dbgs() << ", " << TempRegFlags << ")\n");
975 break;
976 }
977
978 case GIR_AddImm: {
979 int64_t InsnID = MatchTable[CurrentIdx++];
980 int64_t Imm = MatchTable[CurrentIdx++];
981 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
982 OutMIs[InsnID].addImm(Imm);
983 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
984 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
985 << "], " << Imm << ")\n");
986 break;
987 }
988
989 case GIR_ComplexRenderer: {
990 int64_t InsnID = MatchTable[CurrentIdx++];
991 int64_t RendererID = MatchTable[CurrentIdx++];
992 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
993 for (const auto &RenderOpFn : State.Renderers[RendererID])
994 RenderOpFn(OutMIs[InsnID]);
995 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
996 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
997 << InsnID << "], " << RendererID << ")\n");
998 break;
999 }
1000 case GIR_ComplexSubOperandRenderer: {
1001 int64_t InsnID = MatchTable[CurrentIdx++];
1002 int64_t RendererID = MatchTable[CurrentIdx++];
1003 int64_t RenderOpID = MatchTable[CurrentIdx++];
1004 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1005 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
1006 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1007 dbgs() << CurrentIdx
1008 << ": GIR_ComplexSubOperandRenderer(OutMIs["
1009 << InsnID << "], " << RendererID << ", "
1010 << RenderOpID << ")\n");
1011 break;
1012 }
1013
1014 case GIR_CopyConstantAsSImm: {
1015 int64_t NewInsnID = MatchTable[CurrentIdx++];
1016 int64_t OldInsnID = MatchTable[CurrentIdx++];
1017 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1018 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
1019 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
1020 OutMIs[NewInsnID].addImm(
1021 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
1022 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
1023 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
1024 else
1025 llvm_unreachable("Expected Imm or CImm operand");
1026 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1027 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
1028 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1029 break;
1030 }
1031
1032 // TODO: Needs a test case once we have a pattern that uses this.
1033 case GIR_CopyFConstantAsFPImm: {
1034 int64_t NewInsnID = MatchTable[CurrentIdx++];
1035 int64_t OldInsnID = MatchTable[CurrentIdx++];
1036 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
1037 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
1038 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
1039 OutMIs[NewInsnID].addFPImm(
1040 State.MIs[OldInsnID]->getOperand(1).getFPImm());
1041 else
1042 llvm_unreachable("Expected FPImm operand");
1043 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1044 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
1045 << NewInsnID << "], MIs[" << OldInsnID << "])\n");
1046 break;
1047 }
1048
1049 case GIR_CustomRenderer: {
1050 int64_t InsnID = MatchTable[CurrentIdx++];
1051 int64_t OldInsnID = MatchTable[CurrentIdx++];
1052 int64_t RendererFnID = MatchTable[CurrentIdx++];
1053 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1054 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1055 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
1056 << InsnID << "], MIs[" << OldInsnID << "], "
1057 << RendererFnID << ")\n");
1058 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(
1059 OutMIs[InsnID], *State.MIs[OldInsnID],
1060 -1); // Not a source operand of the old instruction.
1061 break;
1062 }
1063 case GIR_CustomOperandRenderer: {
1064 int64_t InsnID = MatchTable[CurrentIdx++];
1065 int64_t OldInsnID = MatchTable[CurrentIdx++];
1066 int64_t OpIdx = MatchTable[CurrentIdx++];
1067 int64_t RendererFnID = MatchTable[CurrentIdx++];
1068 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1069
1070 DEBUG_WITH_TYPE(
1071 TgtInstructionSelector::getName(),
1072 dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs["
1073 << InsnID << "], MIs[" << OldInsnID << "]->getOperand("
1074 << OpIdx << "), "
1075 << RendererFnID << ")\n");
1076 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
1077 *State.MIs[OldInsnID],
1078 OpIdx);
1079 break;
1080 }
1081 case GIR_ConstrainOperandRC: {
1082 int64_t InsnID = MatchTable[CurrentIdx++];
1083 int64_t OpIdx = MatchTable[CurrentIdx++];
1084 int64_t RCEnum = MatchTable[CurrentIdx++];
1085 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1086 MachineInstr &I = *OutMIs[InsnID].getInstr();
1087 MachineFunction &MF = *I.getParent()->getParent();
1088 MachineRegisterInfo &MRI = MF.getRegInfo();
1089 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum);
1090 MachineOperand &MO = I.getOperand(OpIdx);
1091 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO);
1092 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1093 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
1094 << InsnID << "], " << OpIdx << ", " << RCEnum
1095 << ")\n");
1096 break;
1097 }
1098
1099 case GIR_ConstrainSelectedInstOperands: {
1100 int64_t InsnID = MatchTable[CurrentIdx++];
1101 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1102 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
1103 RBI);
1104 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1105 dbgs() << CurrentIdx
1106 << ": GIR_ConstrainSelectedInstOperands(OutMIs["
1107 << InsnID << "])\n");
1108 break;
1109 }
1110
1111 case GIR_MergeMemOperands: {
1112 int64_t InsnID = MatchTable[CurrentIdx++];
1113 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
1114
1115 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1116 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
1117 << InsnID << "]");
1118 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
1119 while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
1120 GIU_MergeMemOperands_EndOfList) {
1121 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1122 dbgs() << ", MIs[" << MergeInsnID << "]");
1123 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
1124 OutMIs[InsnID].addMemOperand(MMO);
1125 }
1126 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
1127 break;
1128 }
1129
1130 case GIR_EraseFromParent: {
1131 int64_t InsnID = MatchTable[CurrentIdx++];
1132 assert(State.MIs[InsnID] &&
1133 "Attempted to erase an undefined instruction");
1134 State.MIs[InsnID]->eraseFromParent();
1135 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1136 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1137 << InsnID << "])\n");
1138 break;
1139 }
1140
1141 case GIR_MakeTempReg: {
1142 int64_t TempRegID = MatchTable[CurrentIdx++];
1143 int64_t TypeID = MatchTable[CurrentIdx++];
1144
1145 State.TempRegisters[TempRegID] =
1146 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1147 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1148 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1149 << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1150 break;
1151 }
1152
1153 case GIR_Coverage: {
1154 int64_t RuleID = MatchTable[CurrentIdx++];
1155 CoverageInfo.setCovered(RuleID);
1156
1157 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1158 dbgs()
1159 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1160 break;
1161 }
1162
1163 case GIR_Done:
1164 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1165 dbgs() << CurrentIdx << ": GIR_Done\n");
1166 propagateFlags(OutMIs);
1167 return true;
1168
1169 default:
1170 llvm_unreachable("Unexpected command");
1171 }
1172 }
1173 }
1174
1175 } // end namespace llvm
1176
1177 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
1178