1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file declares the API for the instruction selector. 10 /// This class is responsible for selecting machine instructions. 11 /// It's implemented by the target. It's used by the InstructionSelect pass. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H 16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H 17 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 20 #include "llvm/CodeGen/GlobalISel/Utils.h" 21 #include "llvm/CodeGen/MachineInstrBuilder.h" 22 #include "llvm/CodeGen/MachineOperand.h" 23 #include "llvm/CodeGen/MachineRegisterInfo.h" 24 #include "llvm/CodeGen/RegisterBankInfo.h" 25 #include "llvm/CodeGen/TargetInstrInfo.h" 26 #include "llvm/CodeGen/TargetOpcodes.h" 27 #include "llvm/CodeGen/TargetRegisterInfo.h" 28 #include "llvm/IR/Constants.h" 29 #include "llvm/IR/DataLayout.h" 30 #include "llvm/Support/CodeGenCoverage.h" 31 #include "llvm/Support/Debug.h" 32 #include "llvm/Support/ErrorHandling.h" 33 #include "llvm/Support/raw_ostream.h" 34 #include <cassert> 35 #include <cstddef> 36 #include <cstdint> 37 38 namespace llvm { 39 40 /// GlobalISel PatFrag Predicates 41 enum { 42 GIPFP_I64_Invalid = 0, 43 GIPFP_APInt_Invalid = 0, 44 GIPFP_APFloat_Invalid = 0, 45 GIPFP_MI_Invalid = 0, 46 }; 47 48 template <class TgtInstructionSelector, class PredicateBitset, 49 class ComplexMatcherMemFn, class CustomRendererFn> 50 bool InstructionSelector::executeMatchTable( 51 TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, 52 const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> 53 &ISelInfo, 54 const int64_t *MatchTable, const TargetInstrInfo &TII, 55 MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, 56 const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, 57 CodeGenCoverage &CoverageInfo) const { 58 59 uint64_t CurrentIdx = 0; 60 SmallVector<uint64_t, 4> OnFailResumeAt; 61 62 // Bypass the flag check on the instruction, and only look at the MCInstrDesc. 63 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); 64 65 const uint16_t Flags = State.MIs[0]->getFlags(); 66 67 enum RejectAction { RejectAndGiveUp, RejectAndResume }; 68 auto handleReject = [&]() -> RejectAction { 69 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 70 dbgs() << CurrentIdx << ": Rejected\n"); 71 if (OnFailResumeAt.empty()) 72 return RejectAndGiveUp; 73 CurrentIdx = OnFailResumeAt.pop_back_val(); 74 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 75 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" 76 << OnFailResumeAt.size() << " try-blocks remain)\n"); 77 return RejectAndResume; 78 }; 79 80 auto propagateFlags = [=](NewMIVector &OutMIs) { 81 for (auto MIB : OutMIs) { 82 // Set the NoFPExcept flag when no original matched instruction could 83 // raise an FP exception, but the new instruction potentially might. 84 uint16_t MIBFlags = Flags; 85 if (NoFPException && MIB->mayRaiseFPException()) 86 MIBFlags |= MachineInstr::NoFPExcept; 87 MIB.setMIFlags(MIBFlags); 88 } 89 90 return true; 91 }; 92 93 while (true) { 94 assert(CurrentIdx != ~0u && "Invalid MatchTable index"); 95 int64_t MatcherOpcode = MatchTable[CurrentIdx++]; 96 switch (MatcherOpcode) { 97 case GIM_Try: { 98 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 99 dbgs() << CurrentIdx << ": Begin try-block\n"); 100 OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); 101 break; 102 } 103 104 case GIM_RecordInsn: { 105 int64_t NewInsnID = MatchTable[CurrentIdx++]; 106 int64_t InsnID = MatchTable[CurrentIdx++]; 107 int64_t OpIdx = MatchTable[CurrentIdx++]; 108 109 // As an optimisation we require that MIs[0] is always the root. Refuse 110 // any attempt to modify it. 111 assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); 112 113 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 114 if (!MO.isReg()) { 115 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 116 dbgs() << CurrentIdx << ": Not a register\n"); 117 if (handleReject() == RejectAndGiveUp) 118 return false; 119 break; 120 } 121 if (MO.getReg().isPhysical()) { 122 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 123 dbgs() << CurrentIdx << ": Is a physical register\n"); 124 if (handleReject() == RejectAndGiveUp) 125 return false; 126 break; 127 } 128 129 MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); 130 if ((size_t)NewInsnID < State.MIs.size()) 131 State.MIs[NewInsnID] = NewMI; 132 else { 133 assert((size_t)NewInsnID == State.MIs.size() && 134 "Expected to store MIs in order"); 135 State.MIs.push_back(NewMI); 136 } 137 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 138 dbgs() << CurrentIdx << ": MIs[" << NewInsnID 139 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx 140 << ")\n"); 141 break; 142 } 143 144 case GIM_CheckFeatures: { 145 int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; 146 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 147 dbgs() << CurrentIdx 148 << ": GIM_CheckFeatures(ExpectedBitsetID=" 149 << ExpectedBitsetID << ")\n"); 150 if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != 151 ISelInfo.FeatureBitsets[ExpectedBitsetID]) { 152 if (handleReject() == RejectAndGiveUp) 153 return false; 154 } 155 break; 156 } 157 158 case GIM_CheckOpcode: 159 case GIM_CheckOpcodeIsEither: { 160 int64_t InsnID = MatchTable[CurrentIdx++]; 161 int64_t Expected0 = MatchTable[CurrentIdx++]; 162 int64_t Expected1 = -1; 163 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 164 Expected1 = MatchTable[CurrentIdx++]; 165 166 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 167 unsigned Opcode = State.MIs[InsnID]->getOpcode(); 168 169 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 170 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID 171 << "], ExpectedOpcode=" << Expected0; 172 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 173 dbgs() << " || " << Expected1; 174 dbgs() << ") // Got=" << Opcode << "\n"; 175 ); 176 177 if (Opcode != Expected0 && Opcode != Expected1) { 178 if (handleReject() == RejectAndGiveUp) 179 return false; 180 } 181 break; 182 } 183 case GIM_SwitchOpcode: { 184 int64_t InsnID = MatchTable[CurrentIdx++]; 185 int64_t LowerBound = MatchTable[CurrentIdx++]; 186 int64_t UpperBound = MatchTable[CurrentIdx++]; 187 int64_t Default = MatchTable[CurrentIdx++]; 188 189 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 190 const int64_t Opcode = State.MIs[InsnID]->getOpcode(); 191 192 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { 193 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" 194 << LowerBound << ", " << UpperBound << "), Default=" << Default 195 << ", JumpTable...) // Got=" << Opcode << "\n"; 196 }); 197 if (Opcode < LowerBound || UpperBound <= Opcode) { 198 CurrentIdx = Default; 199 break; 200 } 201 CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; 202 if (!CurrentIdx) { 203 CurrentIdx = Default; 204 break; 205 } 206 OnFailResumeAt.push_back(Default); 207 break; 208 } 209 210 case GIM_SwitchType: { 211 int64_t InsnID = MatchTable[CurrentIdx++]; 212 int64_t OpIdx = MatchTable[CurrentIdx++]; 213 int64_t LowerBound = MatchTable[CurrentIdx++]; 214 int64_t UpperBound = MatchTable[CurrentIdx++]; 215 int64_t Default = MatchTable[CurrentIdx++]; 216 217 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 218 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 219 220 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { 221 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID 222 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " 223 << UpperBound << "), Default=" << Default 224 << ", JumpTable...) // Got="; 225 if (!MO.isReg()) 226 dbgs() << "Not a VReg\n"; 227 else 228 dbgs() << MRI.getType(MO.getReg()) << "\n"; 229 }); 230 if (!MO.isReg()) { 231 CurrentIdx = Default; 232 break; 233 } 234 const LLT Ty = MRI.getType(MO.getReg()); 235 const auto TyI = ISelInfo.TypeIDMap.find(Ty); 236 if (TyI == ISelInfo.TypeIDMap.end()) { 237 CurrentIdx = Default; 238 break; 239 } 240 const int64_t TypeID = TyI->second; 241 if (TypeID < LowerBound || UpperBound <= TypeID) { 242 CurrentIdx = Default; 243 break; 244 } 245 CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; 246 if (!CurrentIdx) { 247 CurrentIdx = Default; 248 break; 249 } 250 OnFailResumeAt.push_back(Default); 251 break; 252 } 253 254 case GIM_CheckNumOperands: { 255 int64_t InsnID = MatchTable[CurrentIdx++]; 256 int64_t Expected = MatchTable[CurrentIdx++]; 257 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 258 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" 259 << InsnID << "], Expected=" << Expected << ")\n"); 260 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 261 if (State.MIs[InsnID]->getNumOperands() != Expected) { 262 if (handleReject() == RejectAndGiveUp) 263 return false; 264 } 265 break; 266 } 267 case GIM_CheckI64ImmPredicate: 268 case GIM_CheckImmOperandPredicate: { 269 int64_t InsnID = MatchTable[CurrentIdx++]; 270 int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate 271 ? MatchTable[CurrentIdx++] 272 : 1; 273 int64_t Predicate = MatchTable[CurrentIdx++]; 274 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 275 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" 276 << InsnID << "]->getOperand(" << OpIdx 277 << "), Predicate=" << Predicate << ")\n"); 278 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 279 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || 280 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && 281 "Expected immediate operand"); 282 assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); 283 int64_t Value = 0; 284 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) 285 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); 286 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) 287 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); 288 else 289 llvm_unreachable("Expected Imm or CImm operand"); 290 291 if (!testImmPredicate_I64(Predicate, Value)) 292 if (handleReject() == RejectAndGiveUp) 293 return false; 294 break; 295 } 296 case GIM_CheckAPIntImmPredicate: { 297 int64_t InsnID = MatchTable[CurrentIdx++]; 298 int64_t Predicate = MatchTable[CurrentIdx++]; 299 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 300 dbgs() 301 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" 302 << InsnID << "], Predicate=" << Predicate << ")\n"); 303 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 304 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 305 "Expected G_CONSTANT"); 306 assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); 307 APInt Value; 308 if (State.MIs[InsnID]->getOperand(1).isCImm()) 309 Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); 310 else 311 llvm_unreachable("Expected Imm or CImm operand"); 312 313 if (!testImmPredicate_APInt(Predicate, Value)) 314 if (handleReject() == RejectAndGiveUp) 315 return false; 316 break; 317 } 318 case GIM_CheckAPFloatImmPredicate: { 319 int64_t InsnID = MatchTable[CurrentIdx++]; 320 int64_t Predicate = MatchTable[CurrentIdx++]; 321 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 322 dbgs() 323 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" 324 << InsnID << "], Predicate=" << Predicate << ")\n"); 325 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 326 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 327 "Expected G_FCONSTANT"); 328 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); 329 assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); 330 APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); 331 332 if (!testImmPredicate_APFloat(Predicate, Value)) 333 if (handleReject() == RejectAndGiveUp) 334 return false; 335 break; 336 } 337 case GIM_CheckIsBuildVectorAllOnes: 338 case GIM_CheckIsBuildVectorAllZeros: { 339 int64_t InsnID = MatchTable[CurrentIdx++]; 340 341 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 342 dbgs() << CurrentIdx 343 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" 344 << InsnID << "])\n"); 345 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 346 347 const MachineInstr *MI = State.MIs[InsnID]; 348 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || 349 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && 350 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC"); 351 352 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { 353 if (!isBuildVectorAllOnes(*MI, MRI)) { 354 if (handleReject() == RejectAndGiveUp) 355 return false; 356 } 357 } else { 358 if (!isBuildVectorAllZeros(*MI, MRI)) { 359 if (handleReject() == RejectAndGiveUp) 360 return false; 361 } 362 } 363 364 break; 365 } 366 case GIM_CheckCxxInsnPredicate: { 367 int64_t InsnID = MatchTable[CurrentIdx++]; 368 int64_t Predicate = MatchTable[CurrentIdx++]; 369 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 370 dbgs() 371 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" 372 << InsnID << "], Predicate=" << Predicate << ")\n"); 373 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 374 assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); 375 376 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], 377 State.RecordedOperands)) 378 if (handleReject() == RejectAndGiveUp) 379 return false; 380 break; 381 } 382 case GIM_CheckHasNoUse: { 383 int64_t InsnID = MatchTable[CurrentIdx++]; 384 385 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 386 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" 387 << InsnID << "]\n"); 388 389 const MachineInstr *MI = State.MIs[InsnID]; 390 assert(MI && "Used insn before defined"); 391 assert(MI->getNumDefs() > 0 && "No defs"); 392 const Register Res = MI->getOperand(0).getReg(); 393 394 if (!MRI.use_nodbg_empty(Res)) { 395 if (handleReject() == RejectAndGiveUp) 396 return false; 397 } 398 399 break; 400 } 401 case GIM_CheckAtomicOrdering: { 402 int64_t InsnID = MatchTable[CurrentIdx++]; 403 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 404 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 405 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" 406 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 407 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 408 if (!State.MIs[InsnID]->hasOneMemOperand()) 409 if (handleReject() == RejectAndGiveUp) 410 return false; 411 412 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 413 if (MMO->getMergedOrdering() != Ordering) 414 if (handleReject() == RejectAndGiveUp) 415 return false; 416 break; 417 } 418 case GIM_CheckAtomicOrderingOrStrongerThan: { 419 int64_t InsnID = MatchTable[CurrentIdx++]; 420 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 421 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 422 dbgs() << CurrentIdx 423 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" 424 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 425 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 426 if (!State.MIs[InsnID]->hasOneMemOperand()) 427 if (handleReject() == RejectAndGiveUp) 428 return false; 429 430 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 431 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering)) 432 if (handleReject() == RejectAndGiveUp) 433 return false; 434 break; 435 } 436 case GIM_CheckAtomicOrderingWeakerThan: { 437 int64_t InsnID = MatchTable[CurrentIdx++]; 438 AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 439 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 440 dbgs() << CurrentIdx 441 << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" 442 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 443 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 444 if (!State.MIs[InsnID]->hasOneMemOperand()) 445 if (handleReject() == RejectAndGiveUp) 446 return false; 447 448 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 449 if (!isStrongerThan(Ordering, MMO->getMergedOrdering())) 450 if (handleReject() == RejectAndGiveUp) 451 return false; 452 break; 453 } 454 case GIM_CheckMemoryAddressSpace: { 455 int64_t InsnID = MatchTable[CurrentIdx++]; 456 int64_t MMOIdx = MatchTable[CurrentIdx++]; 457 // This accepts a list of possible address spaces. 458 const int NumAddrSpace = MatchTable[CurrentIdx++]; 459 460 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 461 if (handleReject() == RejectAndGiveUp) 462 return false; 463 break; 464 } 465 466 // Need to still jump to the end of the list of address spaces if we find 467 // a match earlier. 468 const uint64_t LastIdx = CurrentIdx + NumAddrSpace; 469 470 const MachineMemOperand *MMO 471 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 472 const unsigned MMOAddrSpace = MMO->getAddrSpace(); 473 474 bool Success = false; 475 for (int I = 0; I != NumAddrSpace; ++I) { 476 unsigned AddrSpace = MatchTable[CurrentIdx++]; 477 DEBUG_WITH_TYPE( 478 TgtInstructionSelector::getName(), 479 dbgs() << "addrspace(" << MMOAddrSpace << ") vs " 480 << AddrSpace << '\n'); 481 482 if (AddrSpace == MMOAddrSpace) { 483 Success = true; 484 break; 485 } 486 } 487 488 CurrentIdx = LastIdx; 489 if (!Success && handleReject() == RejectAndGiveUp) 490 return false; 491 break; 492 } 493 case GIM_CheckMemoryAlignment: { 494 int64_t InsnID = MatchTable[CurrentIdx++]; 495 int64_t MMOIdx = MatchTable[CurrentIdx++]; 496 unsigned MinAlign = MatchTable[CurrentIdx++]; 497 498 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 499 500 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 501 if (handleReject() == RejectAndGiveUp) 502 return false; 503 break; 504 } 505 506 MachineMemOperand *MMO 507 = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 508 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 509 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" 510 << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 511 << ")->getAlignment() >= " << MinAlign << ")\n"); 512 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) 513 return false; 514 515 break; 516 } 517 case GIM_CheckMemorySizeEqualTo: { 518 int64_t InsnID = MatchTable[CurrentIdx++]; 519 int64_t MMOIdx = MatchTable[CurrentIdx++]; 520 uint64_t Size = MatchTable[CurrentIdx++]; 521 522 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 523 dbgs() << CurrentIdx 524 << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID 525 << "]->memoperands() + " << MMOIdx 526 << ", Size=" << Size << ")\n"); 527 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 528 529 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 530 if (handleReject() == RejectAndGiveUp) 531 return false; 532 break; 533 } 534 535 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 536 537 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 538 dbgs() << MMO->getSize() << " bytes vs " << Size 539 << " bytes\n"); 540 if (MMO->getSize() != Size) 541 if (handleReject() == RejectAndGiveUp) 542 return false; 543 544 break; 545 } 546 case GIM_CheckMemorySizeEqualToLLT: 547 case GIM_CheckMemorySizeLessThanLLT: 548 case GIM_CheckMemorySizeGreaterThanLLT: { 549 int64_t InsnID = MatchTable[CurrentIdx++]; 550 int64_t MMOIdx = MatchTable[CurrentIdx++]; 551 int64_t OpIdx = MatchTable[CurrentIdx++]; 552 553 DEBUG_WITH_TYPE( 554 TgtInstructionSelector::getName(), 555 dbgs() << CurrentIdx << ": GIM_CheckMemorySize" 556 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT 557 ? "EqualTo" 558 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT 559 ? "GreaterThan" 560 : "LessThan") 561 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 562 << ", OpIdx=" << OpIdx << ")\n"); 563 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 564 565 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 566 if (!MO.isReg()) { 567 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 568 dbgs() << CurrentIdx << ": Not a register\n"); 569 if (handleReject() == RejectAndGiveUp) 570 return false; 571 break; 572 } 573 574 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 575 if (handleReject() == RejectAndGiveUp) 576 return false; 577 break; 578 } 579 580 MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 581 582 unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); 583 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && 584 MMO->getSizeInBits() != Size) { 585 if (handleReject() == RejectAndGiveUp) 586 return false; 587 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && 588 MMO->getSizeInBits() >= Size) { 589 if (handleReject() == RejectAndGiveUp) 590 return false; 591 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && 592 MMO->getSizeInBits() <= Size) 593 if (handleReject() == RejectAndGiveUp) 594 return false; 595 596 break; 597 } 598 case GIM_CheckType: { 599 int64_t InsnID = MatchTable[CurrentIdx++]; 600 int64_t OpIdx = MatchTable[CurrentIdx++]; 601 int64_t TypeID = MatchTable[CurrentIdx++]; 602 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 603 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID 604 << "]->getOperand(" << OpIdx 605 << "), TypeID=" << TypeID << ")\n"); 606 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 607 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 608 if (!MO.isReg() || 609 MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { 610 if (handleReject() == RejectAndGiveUp) 611 return false; 612 } 613 break; 614 } 615 case GIM_CheckPointerToAny: { 616 int64_t InsnID = MatchTable[CurrentIdx++]; 617 int64_t OpIdx = MatchTable[CurrentIdx++]; 618 uint64_t SizeInBits = MatchTable[CurrentIdx++]; 619 620 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 621 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" 622 << InsnID << "]->getOperand(" << OpIdx 623 << "), SizeInBits=" << SizeInBits << ")\n"); 624 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 625 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 626 const LLT Ty = MRI.getType(MO.getReg()); 627 628 // iPTR must be looked up in the target. 629 if (SizeInBits == 0) { 630 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); 631 const unsigned AddrSpace = Ty.getAddressSpace(); 632 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); 633 } 634 635 assert(SizeInBits != 0 && "Pointer size must be known"); 636 637 if (MO.isReg()) { 638 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) 639 if (handleReject() == RejectAndGiveUp) 640 return false; 641 } else if (handleReject() == RejectAndGiveUp) 642 return false; 643 644 break; 645 } 646 case GIM_RecordNamedOperand: { 647 int64_t InsnID = MatchTable[CurrentIdx++]; 648 int64_t OpIdx = MatchTable[CurrentIdx++]; 649 uint64_t StoreIdx = MatchTable[CurrentIdx++]; 650 651 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 652 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" 653 << InsnID << "]->getOperand(" << OpIdx 654 << "), StoreIdx=" << StoreIdx << ")\n"); 655 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 656 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); 657 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); 658 break; 659 } 660 case GIM_CheckRegBankForClass: { 661 int64_t InsnID = MatchTable[CurrentIdx++]; 662 int64_t OpIdx = MatchTable[CurrentIdx++]; 663 int64_t RCEnum = MatchTable[CurrentIdx++]; 664 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 665 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" 666 << InsnID << "]->getOperand(" << OpIdx 667 << "), RCEnum=" << RCEnum << ")\n"); 668 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 669 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 670 if (!MO.isReg() || 671 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), 672 MRI.getType(MO.getReg())) != 673 RBI.getRegBank(MO.getReg(), MRI, TRI)) { 674 if (handleReject() == RejectAndGiveUp) 675 return false; 676 } 677 break; 678 } 679 680 case GIM_CheckComplexPattern: { 681 int64_t InsnID = MatchTable[CurrentIdx++]; 682 int64_t OpIdx = MatchTable[CurrentIdx++]; 683 int64_t RendererID = MatchTable[CurrentIdx++]; 684 int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; 685 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 686 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID 687 << "] = GIM_CheckComplexPattern(MIs[" << InsnID 688 << "]->getOperand(" << OpIdx 689 << "), ComplexPredicateID=" << ComplexPredicateID 690 << ")\n"); 691 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 692 // FIXME: Use std::invoke() when it's available. 693 ComplexRendererFns Renderer = 694 (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( 695 State.MIs[InsnID]->getOperand(OpIdx)); 696 if (Renderer) 697 State.Renderers[RendererID] = *Renderer; 698 else 699 if (handleReject() == RejectAndGiveUp) 700 return false; 701 break; 702 } 703 704 case GIM_CheckConstantInt: { 705 int64_t InsnID = MatchTable[CurrentIdx++]; 706 int64_t OpIdx = MatchTable[CurrentIdx++]; 707 int64_t Value = MatchTable[CurrentIdx++]; 708 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 709 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" 710 << InsnID << "]->getOperand(" << OpIdx 711 << "), Value=" << Value << ")\n"); 712 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 713 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 714 if (MO.isReg()) { 715 // isOperandImmEqual() will sign-extend to 64-bits, so should we. 716 LLT Ty = MRI.getType(MO.getReg()); 717 Value = SignExtend64(Value, Ty.getSizeInBits()); 718 719 if (!isOperandImmEqual(MO, Value, MRI)) { 720 if (handleReject() == RejectAndGiveUp) 721 return false; 722 } 723 } else if (handleReject() == RejectAndGiveUp) 724 return false; 725 726 break; 727 } 728 729 case GIM_CheckLiteralInt: { 730 int64_t InsnID = MatchTable[CurrentIdx++]; 731 int64_t OpIdx = MatchTable[CurrentIdx++]; 732 int64_t Value = MatchTable[CurrentIdx++]; 733 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 734 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" 735 << InsnID << "]->getOperand(" << OpIdx 736 << "), Value=" << Value << ")\n"); 737 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 738 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 739 if (MO.isImm() && MO.getImm() == Value) 740 break; 741 742 if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) 743 break; 744 745 if (handleReject() == RejectAndGiveUp) 746 return false; 747 748 break; 749 } 750 751 case GIM_CheckIntrinsicID: { 752 int64_t InsnID = MatchTable[CurrentIdx++]; 753 int64_t OpIdx = MatchTable[CurrentIdx++]; 754 int64_t Value = MatchTable[CurrentIdx++]; 755 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 756 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" 757 << InsnID << "]->getOperand(" << OpIdx 758 << "), Value=" << Value << ")\n"); 759 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 760 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 761 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) 762 if (handleReject() == RejectAndGiveUp) 763 return false; 764 break; 765 } 766 case GIM_CheckCmpPredicate: { 767 int64_t InsnID = MatchTable[CurrentIdx++]; 768 int64_t OpIdx = MatchTable[CurrentIdx++]; 769 int64_t Value = MatchTable[CurrentIdx++]; 770 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 771 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" 772 << InsnID << "]->getOperand(" << OpIdx 773 << "), Value=" << Value << ")\n"); 774 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 775 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 776 if (!MO.isPredicate() || MO.getPredicate() != Value) 777 if (handleReject() == RejectAndGiveUp) 778 return false; 779 break; 780 } 781 case GIM_CheckIsMBB: { 782 int64_t InsnID = MatchTable[CurrentIdx++]; 783 int64_t OpIdx = MatchTable[CurrentIdx++]; 784 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 785 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID 786 << "]->getOperand(" << OpIdx << "))\n"); 787 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 788 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { 789 if (handleReject() == RejectAndGiveUp) 790 return false; 791 } 792 break; 793 } 794 case GIM_CheckIsImm: { 795 int64_t InsnID = MatchTable[CurrentIdx++]; 796 int64_t OpIdx = MatchTable[CurrentIdx++]; 797 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 798 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID 799 << "]->getOperand(" << OpIdx << "))\n"); 800 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 801 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { 802 if (handleReject() == RejectAndGiveUp) 803 return false; 804 } 805 break; 806 } 807 case GIM_CheckIsSafeToFold: { 808 int64_t InsnID = MatchTable[CurrentIdx++]; 809 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 810 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" 811 << InsnID << "])\n"); 812 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 813 if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { 814 if (handleReject() == RejectAndGiveUp) 815 return false; 816 } 817 break; 818 } 819 case GIM_CheckIsSameOperand: { 820 int64_t InsnID = MatchTable[CurrentIdx++]; 821 int64_t OpIdx = MatchTable[CurrentIdx++]; 822 int64_t OtherInsnID = MatchTable[CurrentIdx++]; 823 int64_t OtherOpIdx = MatchTable[CurrentIdx++]; 824 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 825 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" 826 << InsnID << "][" << OpIdx << "], MIs[" 827 << OtherInsnID << "][" << OtherOpIdx << "])\n"); 828 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 829 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); 830 if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( 831 State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { 832 if (handleReject() == RejectAndGiveUp) 833 return false; 834 } 835 break; 836 } 837 case GIM_Reject: 838 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 839 dbgs() << CurrentIdx << ": GIM_Reject\n"); 840 if (handleReject() == RejectAndGiveUp) 841 return false; 842 break; 843 844 case GIR_MutateOpcode: { 845 int64_t OldInsnID = MatchTable[CurrentIdx++]; 846 uint64_t NewInsnID = MatchTable[CurrentIdx++]; 847 int64_t NewOpcode = MatchTable[CurrentIdx++]; 848 if (NewInsnID >= OutMIs.size()) 849 OutMIs.resize(NewInsnID + 1); 850 851 OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), 852 State.MIs[OldInsnID]); 853 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); 854 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 855 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" 856 << NewInsnID << "], MIs[" << OldInsnID << "], " 857 << NewOpcode << ")\n"); 858 break; 859 } 860 861 case GIR_BuildMI: { 862 uint64_t NewInsnID = MatchTable[CurrentIdx++]; 863 int64_t Opcode = MatchTable[CurrentIdx++]; 864 if (NewInsnID >= OutMIs.size()) 865 OutMIs.resize(NewInsnID + 1); 866 867 OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], 868 MIMetadata(*State.MIs[0]), TII.get(Opcode)); 869 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 870 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" 871 << NewInsnID << "], " << Opcode << ")\n"); 872 break; 873 } 874 875 case GIR_Copy: { 876 int64_t NewInsnID = MatchTable[CurrentIdx++]; 877 int64_t OldInsnID = MatchTable[CurrentIdx++]; 878 int64_t OpIdx = MatchTable[CurrentIdx++]; 879 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 880 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); 881 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 882 dbgs() 883 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID 884 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); 885 break; 886 } 887 888 case GIR_CopyOrAddZeroReg: { 889 int64_t NewInsnID = MatchTable[CurrentIdx++]; 890 int64_t OldInsnID = MatchTable[CurrentIdx++]; 891 int64_t OpIdx = MatchTable[CurrentIdx++]; 892 int64_t ZeroReg = MatchTable[CurrentIdx++]; 893 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 894 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); 895 if (isOperandImmEqual(MO, 0, MRI)) 896 OutMIs[NewInsnID].addReg(ZeroReg); 897 else 898 OutMIs[NewInsnID].add(MO); 899 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 900 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" 901 << NewInsnID << "], MIs[" << OldInsnID << "], " 902 << OpIdx << ", " << ZeroReg << ")\n"); 903 break; 904 } 905 906 case GIR_CopySubReg: { 907 int64_t NewInsnID = MatchTable[CurrentIdx++]; 908 int64_t OldInsnID = MatchTable[CurrentIdx++]; 909 int64_t OpIdx = MatchTable[CurrentIdx++]; 910 int64_t SubRegIdx = MatchTable[CurrentIdx++]; 911 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 912 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 913 0, SubRegIdx); 914 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 915 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" 916 << NewInsnID << "], MIs[" << OldInsnID << "], " 917 << OpIdx << ", " << SubRegIdx << ")\n"); 918 break; 919 } 920 921 case GIR_AddImplicitDef: { 922 int64_t InsnID = MatchTable[CurrentIdx++]; 923 int64_t RegNum = MatchTable[CurrentIdx++]; 924 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 925 OutMIs[InsnID].addDef(RegNum, RegState::Implicit); 926 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 927 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" 928 << InsnID << "], " << RegNum << ")\n"); 929 break; 930 } 931 932 case GIR_AddImplicitUse: { 933 int64_t InsnID = MatchTable[CurrentIdx++]; 934 int64_t RegNum = MatchTable[CurrentIdx++]; 935 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 936 OutMIs[InsnID].addUse(RegNum, RegState::Implicit); 937 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 938 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" 939 << InsnID << "], " << RegNum << ")\n"); 940 break; 941 } 942 943 case GIR_AddRegister: { 944 int64_t InsnID = MatchTable[CurrentIdx++]; 945 int64_t RegNum = MatchTable[CurrentIdx++]; 946 uint64_t RegFlags = MatchTable[CurrentIdx++]; 947 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 948 OutMIs[InsnID].addReg(RegNum, RegFlags); 949 DEBUG_WITH_TYPE( 950 TgtInstructionSelector::getName(), 951 dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" 952 << InsnID << "], " << RegNum << ", " << RegFlags << ")\n"); 953 break; 954 } 955 956 case GIR_AddTempRegister: 957 case GIR_AddTempSubRegister: { 958 int64_t InsnID = MatchTable[CurrentIdx++]; 959 int64_t TempRegID = MatchTable[CurrentIdx++]; 960 uint64_t TempRegFlags = MatchTable[CurrentIdx++]; 961 unsigned SubReg = 0; 962 if (MatcherOpcode == GIR_AddTempSubRegister) 963 SubReg = MatchTable[CurrentIdx++]; 964 965 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 966 967 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, SubReg); 968 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 969 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" 970 << InsnID << "], TempRegisters[" << TempRegID 971 << "]"; 972 if (SubReg) 973 dbgs() << '.' << TRI.getSubRegIndexName(SubReg); 974 dbgs() << ", " << TempRegFlags << ")\n"); 975 break; 976 } 977 978 case GIR_AddImm: { 979 int64_t InsnID = MatchTable[CurrentIdx++]; 980 int64_t Imm = MatchTable[CurrentIdx++]; 981 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 982 OutMIs[InsnID].addImm(Imm); 983 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 984 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID 985 << "], " << Imm << ")\n"); 986 break; 987 } 988 989 case GIR_ComplexRenderer: { 990 int64_t InsnID = MatchTable[CurrentIdx++]; 991 int64_t RendererID = MatchTable[CurrentIdx++]; 992 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 993 for (const auto &RenderOpFn : State.Renderers[RendererID]) 994 RenderOpFn(OutMIs[InsnID]); 995 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 996 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" 997 << InsnID << "], " << RendererID << ")\n"); 998 break; 999 } 1000 case GIR_ComplexSubOperandRenderer: { 1001 int64_t InsnID = MatchTable[CurrentIdx++]; 1002 int64_t RendererID = MatchTable[CurrentIdx++]; 1003 int64_t RenderOpID = MatchTable[CurrentIdx++]; 1004 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1005 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); 1006 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1007 dbgs() << CurrentIdx 1008 << ": GIR_ComplexSubOperandRenderer(OutMIs[" 1009 << InsnID << "], " << RendererID << ", " 1010 << RenderOpID << ")\n"); 1011 break; 1012 } 1013 1014 case GIR_CopyConstantAsSImm: { 1015 int64_t NewInsnID = MatchTable[CurrentIdx++]; 1016 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1017 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1018 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); 1019 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { 1020 OutMIs[NewInsnID].addImm( 1021 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); 1022 } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) 1023 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); 1024 else 1025 llvm_unreachable("Expected Imm or CImm operand"); 1026 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1027 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" 1028 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1029 break; 1030 } 1031 1032 // TODO: Needs a test case once we have a pattern that uses this. 1033 case GIR_CopyFConstantAsFPImm: { 1034 int64_t NewInsnID = MatchTable[CurrentIdx++]; 1035 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1036 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1037 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); 1038 if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) 1039 OutMIs[NewInsnID].addFPImm( 1040 State.MIs[OldInsnID]->getOperand(1).getFPImm()); 1041 else 1042 llvm_unreachable("Expected FPImm operand"); 1043 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1044 dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" 1045 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1046 break; 1047 } 1048 1049 case GIR_CustomRenderer: { 1050 int64_t InsnID = MatchTable[CurrentIdx++]; 1051 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1052 int64_t RendererFnID = MatchTable[CurrentIdx++]; 1053 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1054 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1055 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" 1056 << InsnID << "], MIs[" << OldInsnID << "], " 1057 << RendererFnID << ")\n"); 1058 (ISel.*ISelInfo.CustomRenderers[RendererFnID])( 1059 OutMIs[InsnID], *State.MIs[OldInsnID], 1060 -1); // Not a source operand of the old instruction. 1061 break; 1062 } 1063 case GIR_CustomOperandRenderer: { 1064 int64_t InsnID = MatchTable[CurrentIdx++]; 1065 int64_t OldInsnID = MatchTable[CurrentIdx++]; 1066 int64_t OpIdx = MatchTable[CurrentIdx++]; 1067 int64_t RendererFnID = MatchTable[CurrentIdx++]; 1068 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1069 1070 DEBUG_WITH_TYPE( 1071 TgtInstructionSelector::getName(), 1072 dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs[" 1073 << InsnID << "], MIs[" << OldInsnID << "]->getOperand(" 1074 << OpIdx << "), " 1075 << RendererFnID << ")\n"); 1076 (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], 1077 *State.MIs[OldInsnID], 1078 OpIdx); 1079 break; 1080 } 1081 case GIR_ConstrainOperandRC: { 1082 int64_t InsnID = MatchTable[CurrentIdx++]; 1083 int64_t OpIdx = MatchTable[CurrentIdx++]; 1084 int64_t RCEnum = MatchTable[CurrentIdx++]; 1085 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1086 MachineInstr &I = *OutMIs[InsnID].getInstr(); 1087 MachineFunction &MF = *I.getParent()->getParent(); 1088 MachineRegisterInfo &MRI = MF.getRegInfo(); 1089 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum); 1090 MachineOperand &MO = I.getOperand(OpIdx); 1091 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO); 1092 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1093 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" 1094 << InsnID << "], " << OpIdx << ", " << RCEnum 1095 << ")\n"); 1096 break; 1097 } 1098 1099 case GIR_ConstrainSelectedInstOperands: { 1100 int64_t InsnID = MatchTable[CurrentIdx++]; 1101 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1102 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, 1103 RBI); 1104 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1105 dbgs() << CurrentIdx 1106 << ": GIR_ConstrainSelectedInstOperands(OutMIs[" 1107 << InsnID << "])\n"); 1108 break; 1109 } 1110 1111 case GIR_MergeMemOperands: { 1112 int64_t InsnID = MatchTable[CurrentIdx++]; 1113 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1114 1115 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1116 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" 1117 << InsnID << "]"); 1118 int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; 1119 while ((MergeInsnID = MatchTable[CurrentIdx++]) != 1120 GIU_MergeMemOperands_EndOfList) { 1121 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1122 dbgs() << ", MIs[" << MergeInsnID << "]"); 1123 for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) 1124 OutMIs[InsnID].addMemOperand(MMO); 1125 } 1126 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); 1127 break; 1128 } 1129 1130 case GIR_EraseFromParent: { 1131 int64_t InsnID = MatchTable[CurrentIdx++]; 1132 assert(State.MIs[InsnID] && 1133 "Attempted to erase an undefined instruction"); 1134 State.MIs[InsnID]->eraseFromParent(); 1135 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1136 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" 1137 << InsnID << "])\n"); 1138 break; 1139 } 1140 1141 case GIR_MakeTempReg: { 1142 int64_t TempRegID = MatchTable[CurrentIdx++]; 1143 int64_t TypeID = MatchTable[CurrentIdx++]; 1144 1145 State.TempRegisters[TempRegID] = 1146 MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); 1147 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1148 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID 1149 << "] = GIR_MakeTempReg(" << TypeID << ")\n"); 1150 break; 1151 } 1152 1153 case GIR_Coverage: { 1154 int64_t RuleID = MatchTable[CurrentIdx++]; 1155 CoverageInfo.setCovered(RuleID); 1156 1157 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1158 dbgs() 1159 << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); 1160 break; 1161 } 1162 1163 case GIR_Done: 1164 DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), 1165 dbgs() << CurrentIdx << ": GIR_Done\n"); 1166 propagateFlags(OutMIs); 1167 return true; 1168 1169 default: 1170 llvm_unreachable("Unexpected command"); 1171 } 1172 } 1173 } 1174 1175 } // end namespace llvm 1176 1177 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H 1178