1 //===- llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h -------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file This file implements GIMatchTableExecutor's `executeMatchTable` 10 /// function. This is implemented in a separate file because the function is 11 /// quite large. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 16 #define LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 17 18 #include "llvm/ADT/SmallVector.h" 19 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h" 20 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" 21 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 22 #include "llvm/CodeGen/GlobalISel/Utils.h" 23 #include "llvm/CodeGen/MachineInstrBuilder.h" 24 #include "llvm/CodeGen/MachineOperand.h" 25 #include "llvm/CodeGen/MachineRegisterInfo.h" 26 #include "llvm/CodeGen/RegisterBankInfo.h" 27 #include "llvm/CodeGen/TargetInstrInfo.h" 28 #include "llvm/CodeGen/TargetOpcodes.h" 29 #include "llvm/CodeGen/TargetRegisterInfo.h" 30 #include "llvm/IR/Constants.h" 31 #include "llvm/IR/DataLayout.h" 32 #include "llvm/IR/Type.h" 33 #include "llvm/Support/CodeGenCoverage.h" 34 #include "llvm/Support/Debug.h" 35 #include "llvm/Support/ErrorHandling.h" 36 #include "llvm/Support/LEB128.h" 37 #include "llvm/Support/raw_ostream.h" 38 #include <cassert> 39 #include <cstddef> 40 #include <cstdint> 41 42 namespace llvm { 43 44 template <class TgtExecutor, class PredicateBitset, class ComplexMatcherMemFn, 45 class CustomRendererFn> 46 bool GIMatchTableExecutor::executeMatchTable( 47 TgtExecutor &Exec, MatcherState &State, 48 const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> 49 &ExecInfo, 50 MachineIRBuilder &Builder, const uint8_t *MatchTable, 51 const TargetInstrInfo &TII, MachineRegisterInfo &MRI, 52 const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, 53 const PredicateBitset &AvailableFeatures, 54 CodeGenCoverage *CoverageInfo) const { 55 56 uint64_t CurrentIdx = 0; 57 SmallVector<uint64_t, 4> OnFailResumeAt; 58 NewMIVector OutMIs; 59 60 GISelChangeObserver *Observer = Builder.getObserver(); 61 // Bypass the flag check on the instruction, and only look at the MCInstrDesc. 62 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException(); 63 64 const uint16_t Flags = State.MIs[0]->getFlags(); 65 66 enum RejectAction { RejectAndGiveUp, RejectAndResume }; 67 auto handleReject = [&]() -> RejectAction { 68 DEBUG_WITH_TYPE(TgtExecutor::getName(), 69 dbgs() << CurrentIdx << ": Rejected\n"); 70 if (OnFailResumeAt.empty()) 71 return RejectAndGiveUp; 72 CurrentIdx = OnFailResumeAt.pop_back_val(); 73 DEBUG_WITH_TYPE(TgtExecutor::getName(), 74 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" 75 << OnFailResumeAt.size() << " try-blocks remain)\n"); 76 return RejectAndResume; 77 }; 78 79 const auto propagateFlags = [&]() { 80 for (auto MIB : OutMIs) { 81 // Set the NoFPExcept flag when no original matched instruction could 82 // raise an FP exception, but the new instruction potentially might. 83 uint16_t MIBFlags = Flags; 84 if (NoFPException && MIB->mayRaiseFPException()) 85 MIBFlags |= MachineInstr::NoFPExcept; 86 if (Observer) 87 Observer->changingInstr(*MIB); 88 MIB.setMIFlags(MIBFlags); 89 if (Observer) 90 Observer->changedInstr(*MIB); 91 } 92 }; 93 94 // If the index is >= 0, it's an index in the type objects generated by 95 // TableGen. If the index is <0, it's an index in the recorded types object. 96 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT { 97 if (Idx >= 0) 98 return ExecInfo.TypeObjects[Idx]; 99 return State.RecordedTypes[1 - Idx]; 100 }; 101 102 const auto readULEB = [&]() { 103 return fastDecodeULEB128(MatchTable, CurrentIdx); 104 }; 105 106 // Convenience function to return a signed value. This avoids 107 // us forgetting to first cast to int8_t before casting to a 108 // wider signed int type. 109 // if we casted uint8 directly to a wider type we'd lose 110 // negative values. 111 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; }; 112 113 const auto readU16 = [&]() { 114 auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx); 115 CurrentIdx += 2; 116 return V; 117 }; 118 119 const auto readU32 = [&]() { 120 auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx); 121 CurrentIdx += 4; 122 return V; 123 }; 124 125 const auto readU64 = [&]() { 126 auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx); 127 CurrentIdx += 8; 128 return V; 129 }; 130 131 const auto eraseImpl = [&](MachineInstr *MI) { 132 // If we're erasing the insertion point, ensure we don't leave a dangling 133 // pointer in the builder. 134 if (Builder.getInsertPt() == MI) 135 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator()); 136 if (Observer) 137 Observer->erasingInstr(*MI); 138 MI->eraseFromParent(); 139 }; 140 141 while (true) { 142 assert(CurrentIdx != ~0u && "Invalid MatchTable index"); 143 uint8_t MatcherOpcode = MatchTable[CurrentIdx++]; 144 switch (MatcherOpcode) { 145 case GIM_Try: { 146 DEBUG_WITH_TYPE(TgtExecutor::getName(), 147 dbgs() << CurrentIdx << ": Begin try-block\n"); 148 OnFailResumeAt.push_back(readU32()); 149 break; 150 } 151 152 case GIM_RecordInsn: 153 case GIM_RecordInsnIgnoreCopies: { 154 uint64_t NewInsnID = readULEB(); 155 uint64_t InsnID = readULEB(); 156 uint64_t OpIdx = readULEB(); 157 158 // As an optimisation we require that MIs[0] is always the root. Refuse 159 // any attempt to modify it. 160 assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); 161 162 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 163 if (!MO.isReg()) { 164 DEBUG_WITH_TYPE(TgtExecutor::getName(), 165 dbgs() << CurrentIdx << ": Not a register\n"); 166 if (handleReject() == RejectAndGiveUp) 167 return false; 168 break; 169 } 170 if (MO.getReg().isPhysical()) { 171 DEBUG_WITH_TYPE(TgtExecutor::getName(), 172 dbgs() << CurrentIdx << ": Is a physical register\n"); 173 if (handleReject() == RejectAndGiveUp) 174 return false; 175 break; 176 } 177 178 MachineInstr *NewMI; 179 if (MatcherOpcode == GIM_RecordInsnIgnoreCopies) 180 NewMI = getDefIgnoringCopies(MO.getReg(), MRI); 181 else 182 NewMI = MRI.getVRegDef(MO.getReg()); 183 184 if ((size_t)NewInsnID < State.MIs.size()) 185 State.MIs[NewInsnID] = NewMI; 186 else { 187 assert((size_t)NewInsnID == State.MIs.size() && 188 "Expected to store MIs in order"); 189 State.MIs.push_back(NewMI); 190 } 191 DEBUG_WITH_TYPE(TgtExecutor::getName(), 192 dbgs() << CurrentIdx << ": MIs[" << NewInsnID 193 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx 194 << ")\n"); 195 break; 196 } 197 198 case GIM_CheckFeatures: { 199 uint16_t ExpectedBitsetID = readU16(); 200 DEBUG_WITH_TYPE(TgtExecutor::getName(), 201 dbgs() << CurrentIdx 202 << ": GIM_CheckFeatures(ExpectedBitsetID=" 203 << ExpectedBitsetID << ")\n"); 204 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) != 205 ExecInfo.FeatureBitsets[ExpectedBitsetID]) { 206 if (handleReject() == RejectAndGiveUp) 207 return false; 208 } 209 break; 210 } 211 case GIM_CheckOpcode: 212 case GIM_CheckOpcodeIsEither: { 213 uint64_t InsnID = readULEB(); 214 uint16_t Expected0 = readU16(); 215 uint16_t Expected1 = -1; 216 if (MatcherOpcode == GIM_CheckOpcodeIsEither) 217 Expected1 = readU16(); 218 219 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 220 unsigned Opcode = State.MIs[InsnID]->getOpcode(); 221 222 DEBUG_WITH_TYPE(TgtExecutor::getName(), 223 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID 224 << "], ExpectedOpcode=" << Expected0; 225 if (MatcherOpcode == GIM_CheckOpcodeIsEither) dbgs() 226 << " || " << Expected1; 227 dbgs() << ") // Got=" << Opcode << "\n";); 228 229 if (Opcode != Expected0 && Opcode != Expected1) { 230 if (handleReject() == RejectAndGiveUp) 231 return false; 232 } 233 break; 234 } 235 case GIM_SwitchOpcode: { 236 uint64_t InsnID = readULEB(); 237 uint16_t LowerBound = readU16(); 238 uint16_t UpperBound = readU16(); 239 uint32_t Default = readU32(); 240 241 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 242 const int64_t Opcode = State.MIs[InsnID]->getOpcode(); 243 244 DEBUG_WITH_TYPE(TgtExecutor::getName(), { 245 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" 246 << LowerBound << ", " << UpperBound << "), Default=" << Default 247 << ", JumpTable...) // Got=" << Opcode << "\n"; 248 }); 249 if (Opcode < LowerBound || UpperBound <= Opcode) { 250 CurrentIdx = Default; 251 break; 252 } 253 const auto EntryIdx = (Opcode - LowerBound); 254 // Each entry is 4 bytes 255 CurrentIdx = 256 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4)); 257 if (!CurrentIdx) { 258 CurrentIdx = Default; 259 break; 260 } 261 OnFailResumeAt.push_back(Default); 262 break; 263 } 264 265 case GIM_SwitchType: { 266 uint64_t InsnID = readULEB(); 267 uint64_t OpIdx = readULEB(); 268 uint16_t LowerBound = readU16(); 269 uint16_t UpperBound = readU16(); 270 int64_t Default = readU32(); 271 272 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 273 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 274 275 DEBUG_WITH_TYPE(TgtExecutor::getName(), { 276 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID 277 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " 278 << UpperBound << "), Default=" << Default 279 << ", JumpTable...) // Got="; 280 if (!MO.isReg()) 281 dbgs() << "Not a VReg\n"; 282 else 283 dbgs() << MRI.getType(MO.getReg()) << "\n"; 284 }); 285 if (!MO.isReg()) { 286 CurrentIdx = Default; 287 break; 288 } 289 const LLT Ty = MRI.getType(MO.getReg()); 290 const auto TyI = ExecInfo.TypeIDMap.find(Ty); 291 if (TyI == ExecInfo.TypeIDMap.end()) { 292 CurrentIdx = Default; 293 break; 294 } 295 const int64_t TypeID = TyI->second; 296 if (TypeID < LowerBound || UpperBound <= TypeID) { 297 CurrentIdx = Default; 298 break; 299 } 300 const auto NumEntry = (TypeID - LowerBound); 301 // Each entry is 4 bytes 302 CurrentIdx = 303 readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4)); 304 if (!CurrentIdx) { 305 CurrentIdx = Default; 306 break; 307 } 308 OnFailResumeAt.push_back(Default); 309 break; 310 } 311 312 case GIM_CheckNumOperands: { 313 uint64_t InsnID = readULEB(); 314 uint64_t Expected = readULEB(); 315 DEBUG_WITH_TYPE(TgtExecutor::getName(), 316 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" 317 << InsnID << "], Expected=" << Expected << ")\n"); 318 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 319 if (State.MIs[InsnID]->getNumOperands() != Expected) { 320 if (handleReject() == RejectAndGiveUp) 321 return false; 322 } 323 break; 324 } 325 case GIM_CheckI64ImmPredicate: 326 case GIM_CheckImmOperandPredicate: { 327 uint64_t InsnID = readULEB(); 328 unsigned OpIdx = 329 MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1; 330 uint16_t Predicate = readU16(); 331 DEBUG_WITH_TYPE(TgtExecutor::getName(), 332 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs[" 333 << InsnID << "]->getOperand(" << OpIdx 334 << "), Predicate=" << Predicate << ")\n"); 335 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 336 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() || 337 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) && 338 "Expected immediate operand"); 339 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 340 int64_t Value = 0; 341 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm()) 342 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue(); 343 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm()) 344 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm(); 345 else 346 llvm_unreachable("Expected Imm or CImm operand"); 347 348 if (!testImmPredicate_I64(Predicate, Value)) 349 if (handleReject() == RejectAndGiveUp) 350 return false; 351 break; 352 } 353 case GIM_CheckAPIntImmPredicate: { 354 uint64_t InsnID = readULEB(); 355 uint16_t Predicate = readU16(); 356 DEBUG_WITH_TYPE(TgtExecutor::getName(), 357 dbgs() 358 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" 359 << InsnID << "], Predicate=" << Predicate << ")\n"); 360 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 361 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 362 "Expected G_CONSTANT"); 363 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 364 if (!State.MIs[InsnID]->getOperand(1).isCImm()) 365 llvm_unreachable("Expected Imm or CImm operand"); 366 367 const APInt &Value = 368 State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); 369 if (!testImmPredicate_APInt(Predicate, Value)) 370 if (handleReject() == RejectAndGiveUp) 371 return false; 372 break; 373 } 374 case GIM_CheckAPFloatImmPredicate: { 375 uint64_t InsnID = readULEB(); 376 uint16_t Predicate = readU16(); 377 DEBUG_WITH_TYPE(TgtExecutor::getName(), 378 dbgs() 379 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" 380 << InsnID << "], Predicate=" << Predicate << ")\n"); 381 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 382 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 383 "Expected G_FCONSTANT"); 384 assert(State.MIs[InsnID]->getOperand(1).isFPImm() && 385 "Expected FPImm operand"); 386 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 387 const APFloat &Value = 388 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); 389 390 if (!testImmPredicate_APFloat(Predicate, Value)) 391 if (handleReject() == RejectAndGiveUp) 392 return false; 393 break; 394 } 395 case GIM_CheckIsBuildVectorAllOnes: 396 case GIM_CheckIsBuildVectorAllZeros: { 397 uint64_t InsnID = readULEB(); 398 399 DEBUG_WITH_TYPE(TgtExecutor::getName(), 400 dbgs() << CurrentIdx 401 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs[" 402 << InsnID << "])\n"); 403 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 404 405 const MachineInstr *MI = State.MIs[InsnID]; 406 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR || 407 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) && 408 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC"); 409 410 if (MatcherOpcode == GIM_CheckIsBuildVectorAllOnes) { 411 if (!isBuildVectorAllOnes(*MI, MRI)) { 412 if (handleReject() == RejectAndGiveUp) 413 return false; 414 } 415 } else { 416 if (!isBuildVectorAllZeros(*MI, MRI)) { 417 if (handleReject() == RejectAndGiveUp) 418 return false; 419 } 420 } 421 422 break; 423 } 424 case GIM_CheckSimplePredicate: { 425 // Note: we don't check for invalid here because this is purely a hook to 426 // allow some executors (such as the combiner) to check arbitrary, 427 // contextless predicates, such as whether a rule is enabled or not. 428 uint16_t Predicate = readU16(); 429 DEBUG_WITH_TYPE(TgtExecutor::getName(), 430 dbgs() << CurrentIdx 431 << ": GIM_CheckSimplePredicate(Predicate=" 432 << Predicate << ")\n"); 433 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 434 if (!testSimplePredicate(Predicate)) { 435 if (handleReject() == RejectAndGiveUp) 436 return false; 437 } 438 break; 439 } 440 case GIM_CheckCxxInsnPredicate: { 441 uint64_t InsnID = readULEB(); 442 uint16_t Predicate = readU16(); 443 DEBUG_WITH_TYPE(TgtExecutor::getName(), 444 dbgs() 445 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" 446 << InsnID << "], Predicate=" << Predicate << ")\n"); 447 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 448 assert(Predicate > GICXXPred_Invalid && "Expected a valid predicate"); 449 450 if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], State)) 451 if (handleReject() == RejectAndGiveUp) 452 return false; 453 break; 454 } 455 case GIM_CheckHasNoUse: { 456 uint64_t InsnID = readULEB(); 457 458 DEBUG_WITH_TYPE(TgtExecutor::getName(), 459 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs[" 460 << InsnID << "]\n"); 461 462 const MachineInstr *MI = State.MIs[InsnID]; 463 assert(MI && "Used insn before defined"); 464 assert(MI->getNumDefs() > 0 && "No defs"); 465 const Register Res = MI->getOperand(0).getReg(); 466 467 if (!MRI.use_nodbg_empty(Res)) { 468 if (handleReject() == RejectAndGiveUp) 469 return false; 470 } 471 break; 472 } 473 case GIM_CheckHasOneUse: { 474 uint64_t InsnID = readULEB(); 475 476 DEBUG_WITH_TYPE(TgtExecutor::getName(), 477 dbgs() << CurrentIdx << ": GIM_CheckHasOneUse(MIs[" 478 << InsnID << "]\n"); 479 480 const MachineInstr *MI = State.MIs[InsnID]; 481 assert(MI && "Used insn before defined"); 482 assert(MI->getNumDefs() > 0 && "No defs"); 483 const Register Res = MI->getOperand(0).getReg(); 484 485 if (!MRI.hasOneNonDBGUse(Res)) { 486 if (handleReject() == RejectAndGiveUp) 487 return false; 488 } 489 break; 490 } 491 case GIM_CheckAtomicOrdering: { 492 uint64_t InsnID = readULEB(); 493 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 494 DEBUG_WITH_TYPE(TgtExecutor::getName(), 495 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" 496 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 497 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 498 if (!State.MIs[InsnID]->hasOneMemOperand()) 499 if (handleReject() == RejectAndGiveUp) 500 return false; 501 502 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 503 if (MMO->getMergedOrdering() != Ordering) 504 if (handleReject() == RejectAndGiveUp) 505 return false; 506 break; 507 } 508 case GIM_CheckAtomicOrderingOrStrongerThan: { 509 uint64_t InsnID = readULEB(); 510 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 511 DEBUG_WITH_TYPE(TgtExecutor::getName(), 512 dbgs() << CurrentIdx 513 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" 514 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 515 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 516 if (!State.MIs[InsnID]->hasOneMemOperand()) 517 if (handleReject() == RejectAndGiveUp) 518 return false; 519 520 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 521 if (!isAtLeastOrStrongerThan(MMO->getMergedOrdering(), Ordering)) 522 if (handleReject() == RejectAndGiveUp) 523 return false; 524 break; 525 } 526 case GIM_CheckAtomicOrderingWeakerThan: { 527 uint64_t InsnID = readULEB(); 528 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; 529 DEBUG_WITH_TYPE(TgtExecutor::getName(), 530 dbgs() << CurrentIdx 531 << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" 532 << InsnID << "], " << (uint64_t)Ordering << ")\n"); 533 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 534 if (!State.MIs[InsnID]->hasOneMemOperand()) 535 if (handleReject() == RejectAndGiveUp) 536 return false; 537 538 for (const auto &MMO : State.MIs[InsnID]->memoperands()) 539 if (!isStrongerThan(Ordering, MMO->getMergedOrdering())) 540 if (handleReject() == RejectAndGiveUp) 541 return false; 542 break; 543 } 544 case GIM_CheckMemoryAddressSpace: { 545 uint64_t InsnID = readULEB(); 546 uint64_t MMOIdx = readULEB(); 547 // This accepts a list of possible address spaces. 548 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++]; 549 550 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 551 if (handleReject() == RejectAndGiveUp) 552 return false; 553 break; 554 } 555 556 // Need to still jump to the end of the list of address spaces if we find 557 // a match earlier. 558 const uint64_t LastIdx = CurrentIdx + NumAddrSpace; 559 560 const MachineMemOperand *MMO = 561 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 562 const unsigned MMOAddrSpace = MMO->getAddrSpace(); 563 564 bool Success = false; 565 for (unsigned I = 0; I != NumAddrSpace; ++I) { 566 uint64_t AddrSpace = readULEB(); 567 DEBUG_WITH_TYPE(TgtExecutor::getName(), 568 dbgs() << "addrspace(" << MMOAddrSpace << ") vs " 569 << AddrSpace << '\n'); 570 571 if (AddrSpace == MMOAddrSpace) { 572 Success = true; 573 break; 574 } 575 } 576 577 CurrentIdx = LastIdx; 578 if (!Success && handleReject() == RejectAndGiveUp) 579 return false; 580 break; 581 } 582 case GIM_CheckMemoryAlignment: { 583 uint64_t InsnID = readULEB(); 584 uint64_t MMOIdx = readULEB(); 585 uint64_t MinAlign = MatchTable[CurrentIdx++]; 586 587 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 588 589 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 590 if (handleReject() == RejectAndGiveUp) 591 return false; 592 break; 593 } 594 595 MachineMemOperand *MMO = 596 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 597 DEBUG_WITH_TYPE(TgtExecutor::getName(), 598 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment" 599 << "(MIs[" << InsnID << "]->memoperands() + " 600 << MMOIdx << ")->getAlignment() >= " << MinAlign 601 << ")\n"); 602 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp) 603 return false; 604 605 break; 606 } 607 case GIM_CheckMemorySizeEqualTo: { 608 uint64_t InsnID = readULEB(); 609 uint64_t MMOIdx = readULEB(); 610 uint32_t Size = readU32(); 611 612 DEBUG_WITH_TYPE(TgtExecutor::getName(), 613 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs[" 614 << InsnID << "]->memoperands() + " << MMOIdx 615 << ", Size=" << Size << ")\n"); 616 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 617 618 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 619 if (handleReject() == RejectAndGiveUp) 620 return false; 621 break; 622 } 623 624 MachineMemOperand *MMO = 625 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 626 627 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << MMO->getSize() 628 << " bytes vs " << Size 629 << " bytes\n"); 630 if (MMO->getSize() != Size) 631 if (handleReject() == RejectAndGiveUp) 632 return false; 633 634 break; 635 } 636 case GIM_CheckMemorySizeEqualToLLT: 637 case GIM_CheckMemorySizeLessThanLLT: 638 case GIM_CheckMemorySizeGreaterThanLLT: { 639 uint64_t InsnID = readULEB(); 640 uint64_t MMOIdx = readULEB(); 641 uint64_t OpIdx = readULEB(); 642 643 DEBUG_WITH_TYPE( 644 TgtExecutor::getName(), 645 dbgs() << CurrentIdx << ": GIM_CheckMemorySize" 646 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT ? "EqualTo" 647 : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT 648 ? "GreaterThan" 649 : "LessThan") 650 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx 651 << ", OpIdx=" << OpIdx << ")\n"); 652 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 653 654 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 655 if (!MO.isReg()) { 656 DEBUG_WITH_TYPE(TgtExecutor::getName(), 657 dbgs() << CurrentIdx << ": Not a register\n"); 658 if (handleReject() == RejectAndGiveUp) 659 return false; 660 break; 661 } 662 663 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { 664 if (handleReject() == RejectAndGiveUp) 665 return false; 666 break; 667 } 668 669 MachineMemOperand *MMO = 670 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); 671 672 const TypeSize Size = MRI.getType(MO.getReg()).getSizeInBits(); 673 if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && 674 MMO->getSizeInBits() != Size) { 675 if (handleReject() == RejectAndGiveUp) 676 return false; 677 } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && 678 TypeSize::isKnownGE(MMO->getSizeInBits().getValue(), Size)) { 679 if (handleReject() == RejectAndGiveUp) 680 return false; 681 } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && 682 TypeSize::isKnownLE(MMO->getSizeInBits().getValue(), Size)) 683 if (handleReject() == RejectAndGiveUp) 684 return false; 685 686 break; 687 } 688 case GIM_RootCheckType: 689 case GIM_CheckType: { 690 uint64_t InsnID = (MatcherOpcode == GIM_RootCheckType) ? 0 : readULEB(); 691 uint64_t OpIdx = readULEB(); 692 int TypeID = readS8(); 693 DEBUG_WITH_TYPE(TgtExecutor::getName(), 694 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID 695 << "]->getOperand(" << OpIdx 696 << "), TypeID=" << TypeID << ")\n"); 697 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 698 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 699 if (!MO.isReg() || MRI.getType(MO.getReg()) != getTypeFromIdx(TypeID)) { 700 if (handleReject() == RejectAndGiveUp) 701 return false; 702 } 703 break; 704 } 705 case GIM_CheckPointerToAny: { 706 uint64_t InsnID = readULEB(); 707 uint64_t OpIdx = readULEB(); 708 uint64_t SizeInBits = readULEB(); 709 710 DEBUG_WITH_TYPE(TgtExecutor::getName(), 711 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" 712 << InsnID << "]->getOperand(" << OpIdx 713 << "), SizeInBits=" << SizeInBits << ")\n"); 714 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 715 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 716 const LLT Ty = MRI.getType(MO.getReg()); 717 718 // iPTR must be looked up in the target. 719 if (SizeInBits == 0) { 720 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); 721 const unsigned AddrSpace = Ty.getAddressSpace(); 722 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); 723 } 724 725 assert(SizeInBits != 0 && "Pointer size must be known"); 726 727 if (MO.isReg()) { 728 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) 729 if (handleReject() == RejectAndGiveUp) 730 return false; 731 } else if (handleReject() == RejectAndGiveUp) 732 return false; 733 734 break; 735 } 736 case GIM_RecordNamedOperand: { 737 uint64_t InsnID = readULEB(); 738 uint64_t OpIdx = readULEB(); 739 uint64_t StoreIdx = readULEB(); 740 741 DEBUG_WITH_TYPE(TgtExecutor::getName(), 742 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" 743 << InsnID << "]->getOperand(" << OpIdx 744 << "), StoreIdx=" << StoreIdx << ")\n"); 745 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 746 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); 747 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); 748 break; 749 } 750 case GIM_RecordRegType: { 751 uint64_t InsnID = readULEB(); 752 uint64_t OpIdx = readULEB(); 753 int TypeIdx = readS8(); 754 755 DEBUG_WITH_TYPE(TgtExecutor::getName(), 756 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs[" 757 << InsnID << "]->getOperand(" << OpIdx 758 << "), TypeIdx=" << TypeIdx << ")\n"); 759 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 760 assert(TypeIdx < 0 && "Temp types always have negative indexes!"); 761 // Indexes start at -1. 762 TypeIdx = 1 - TypeIdx; 763 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx); 764 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx) 765 State.RecordedTypes.resize(TypeIdx + 1, LLT()); 766 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg()); 767 break; 768 } 769 770 case GIM_RootCheckRegBankForClass: 771 case GIM_CheckRegBankForClass: { 772 uint64_t InsnID = 773 (MatcherOpcode == GIM_RootCheckRegBankForClass) ? 0 : readULEB(); 774 uint64_t OpIdx = readULEB(); 775 uint16_t RCEnum = readU16(); 776 DEBUG_WITH_TYPE(TgtExecutor::getName(), 777 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" 778 << InsnID << "]->getOperand(" << OpIdx 779 << "), RCEnum=" << RCEnum << ")\n"); 780 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 781 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 782 if (!MO.isReg() || 783 &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum), 784 MRI.getType(MO.getReg())) != 785 RBI.getRegBank(MO.getReg(), MRI, TRI)) { 786 if (handleReject() == RejectAndGiveUp) 787 return false; 788 } 789 break; 790 } 791 792 case GIM_CheckComplexPattern: { 793 uint64_t InsnID = readULEB(); 794 uint64_t OpIdx = readULEB(); 795 uint16_t RendererID = readU16(); 796 uint16_t ComplexPredicateID = readU16(); 797 DEBUG_WITH_TYPE(TgtExecutor::getName(), 798 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID 799 << "] = GIM_CheckComplexPattern(MIs[" << InsnID 800 << "]->getOperand(" << OpIdx 801 << "), ComplexPredicateID=" << ComplexPredicateID 802 << ")\n"); 803 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 804 // FIXME: Use std::invoke() when it's available. 805 ComplexRendererFns Renderer = 806 (Exec.*ExecInfo.ComplexPredicates[ComplexPredicateID])( 807 State.MIs[InsnID]->getOperand(OpIdx)); 808 if (Renderer) 809 State.Renderers[RendererID] = *Renderer; 810 else if (handleReject() == RejectAndGiveUp) 811 return false; 812 break; 813 } 814 815 case GIM_CheckConstantInt: 816 case GIM_CheckConstantInt8: { 817 const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8); 818 819 uint64_t InsnID = readULEB(); 820 uint64_t OpIdx = readULEB(); 821 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64(); 822 DEBUG_WITH_TYPE(TgtExecutor::getName(), 823 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" 824 << InsnID << "]->getOperand(" << OpIdx 825 << "), Value=" << Value << ")\n"); 826 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 827 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 828 if (MO.isReg()) { 829 // isOperandImmEqual() will sign-extend to 64-bits, so should we. 830 LLT Ty = MRI.getType(MO.getReg()); 831 // If the type is > 64 bits, it can't be a constant int, so we bail 832 // early because SignExtend64 will assert otherwise. 833 if (Ty.getScalarSizeInBits() > 64) { 834 if (handleReject() == RejectAndGiveUp) 835 return false; 836 break; 837 } 838 839 Value = SignExtend64(Value, Ty.getScalarSizeInBits()); 840 if (!isOperandImmEqual(MO, Value, MRI, /*Splat=*/true)) { 841 if (handleReject() == RejectAndGiveUp) 842 return false; 843 } 844 } else if (handleReject() == RejectAndGiveUp) 845 return false; 846 847 break; 848 } 849 850 case GIM_CheckLiteralInt: { 851 uint64_t InsnID = readULEB(); 852 uint64_t OpIdx = readULEB(); 853 int64_t Value = readU64(); 854 DEBUG_WITH_TYPE(TgtExecutor::getName(), 855 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" 856 << InsnID << "]->getOperand(" << OpIdx 857 << "), Value=" << Value << ")\n"); 858 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 859 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 860 if (MO.isImm() && MO.getImm() == Value) 861 break; 862 863 if (MO.isCImm() && MO.getCImm()->equalsInt(Value)) 864 break; 865 866 if (handleReject() == RejectAndGiveUp) 867 return false; 868 869 break; 870 } 871 872 case GIM_CheckIntrinsicID: { 873 uint64_t InsnID = readULEB(); 874 uint64_t OpIdx = readULEB(); 875 uint16_t Value = readU16(); 876 DEBUG_WITH_TYPE(TgtExecutor::getName(), 877 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" 878 << InsnID << "]->getOperand(" << OpIdx 879 << "), Value=" << Value << ")\n"); 880 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 881 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 882 if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) 883 if (handleReject() == RejectAndGiveUp) 884 return false; 885 break; 886 } 887 case GIM_CheckCmpPredicate: { 888 uint64_t InsnID = readULEB(); 889 uint64_t OpIdx = readULEB(); 890 uint16_t Value = readU16(); 891 DEBUG_WITH_TYPE(TgtExecutor::getName(), 892 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs[" 893 << InsnID << "]->getOperand(" << OpIdx 894 << "), Value=" << Value << ")\n"); 895 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 896 MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); 897 if (!MO.isPredicate() || MO.getPredicate() != Value) 898 if (handleReject() == RejectAndGiveUp) 899 return false; 900 break; 901 } 902 case GIM_CheckIsMBB: { 903 uint64_t InsnID = readULEB(); 904 uint64_t OpIdx = readULEB(); 905 DEBUG_WITH_TYPE(TgtExecutor::getName(), 906 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID 907 << "]->getOperand(" << OpIdx << "))\n"); 908 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 909 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { 910 if (handleReject() == RejectAndGiveUp) 911 return false; 912 } 913 break; 914 } 915 case GIM_CheckIsImm: { 916 uint64_t InsnID = readULEB(); 917 uint64_t OpIdx = readULEB(); 918 DEBUG_WITH_TYPE(TgtExecutor::getName(), 919 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID 920 << "]->getOperand(" << OpIdx << "))\n"); 921 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 922 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) { 923 if (handleReject() == RejectAndGiveUp) 924 return false; 925 } 926 break; 927 } 928 case GIM_CheckIsSafeToFold: { 929 uint64_t NumInsn = MatchTable[CurrentIdx++]; 930 DEBUG_WITH_TYPE(TgtExecutor::getName(), 931 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = " 932 << NumInsn << ")\n"); 933 MachineInstr &Root = *State.MIs[0]; 934 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) { 935 if (!isObviouslySafeToFold(*State.MIs[K], Root)) { 936 if (handleReject() == RejectAndGiveUp) 937 return false; 938 } 939 } 940 break; 941 } 942 case GIM_CheckIsSameOperand: 943 case GIM_CheckIsSameOperandIgnoreCopies: { 944 uint64_t InsnID = readULEB(); 945 uint64_t OpIdx = readULEB(); 946 uint64_t OtherInsnID = readULEB(); 947 uint64_t OtherOpIdx = readULEB(); 948 DEBUG_WITH_TYPE(TgtExecutor::getName(), 949 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" 950 << InsnID << "][" << OpIdx << "], MIs[" 951 << OtherInsnID << "][" << OtherOpIdx << "])\n"); 952 assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); 953 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); 954 955 MachineOperand &Op = State.MIs[InsnID]->getOperand(OpIdx); 956 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx); 957 958 if (MatcherOpcode == GIM_CheckIsSameOperandIgnoreCopies) { 959 if (Op.isReg() && OtherOp.isReg()) { 960 if (getSrcRegIgnoringCopies(Op.getReg(), MRI) == 961 getSrcRegIgnoringCopies(OtherOp.getReg(), MRI)) 962 break; 963 } 964 } 965 966 if (!Op.isIdenticalTo(OtherOp)) { 967 if (handleReject() == RejectAndGiveUp) 968 return false; 969 } 970 break; 971 } 972 case GIM_CheckCanReplaceReg: { 973 uint64_t OldInsnID = readULEB(); 974 uint64_t OldOpIdx = readULEB(); 975 uint64_t NewInsnID = readULEB(); 976 uint64_t NewOpIdx = readULEB(); 977 978 DEBUG_WITH_TYPE(TgtExecutor::getName(), 979 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs[" 980 << OldInsnID << "][" << OldOpIdx << "] = MIs[" 981 << NewInsnID << "][" << NewOpIdx << "])\n"); 982 983 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); 984 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg(); 985 if (!canReplaceReg(Old, New, MRI)) { 986 if (handleReject() == RejectAndGiveUp) 987 return false; 988 } 989 break; 990 } 991 case GIM_MIFlags: { 992 uint64_t InsnID = readULEB(); 993 uint32_t Flags = readU32(); 994 995 DEBUG_WITH_TYPE(TgtExecutor::getName(), 996 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID 997 << "], " << Flags << ")\n"); 998 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) { 999 if (handleReject() == RejectAndGiveUp) 1000 return false; 1001 } 1002 break; 1003 } 1004 case GIM_MIFlagsNot: { 1005 uint64_t InsnID = readULEB(); 1006 uint32_t Flags = readU32(); 1007 1008 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1009 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID 1010 << "], " << Flags << ")\n"); 1011 if ((State.MIs[InsnID]->getFlags() & Flags)) { 1012 if (handleReject() == RejectAndGiveUp) 1013 return false; 1014 } 1015 break; 1016 } 1017 case GIM_Reject: 1018 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1019 dbgs() << CurrentIdx << ": GIM_Reject\n"); 1020 if (handleReject() == RejectAndGiveUp) 1021 return false; 1022 break; 1023 case GIR_MutateOpcode: { 1024 uint64_t OldInsnID = readULEB(); 1025 uint64_t NewInsnID = readULEB(); 1026 uint16_t NewOpcode = readU16(); 1027 if (NewInsnID >= OutMIs.size()) 1028 OutMIs.resize(NewInsnID + 1); 1029 1030 MachineInstr *OldMI = State.MIs[OldInsnID]; 1031 if (Observer) 1032 Observer->changingInstr(*OldMI); 1033 OutMIs[NewInsnID] = MachineInstrBuilder(*OldMI->getMF(), OldMI); 1034 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); 1035 if (Observer) 1036 Observer->changedInstr(*OldMI); 1037 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1038 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" 1039 << NewInsnID << "], MIs[" << OldInsnID << "], " 1040 << NewOpcode << ")\n"); 1041 break; 1042 } 1043 1044 case GIR_BuildRootMI: 1045 case GIR_BuildMI: { 1046 uint64_t NewInsnID = (MatcherOpcode == GIR_BuildRootMI) ? 0 : readULEB(); 1047 uint16_t Opcode = readU16(); 1048 if (NewInsnID >= OutMIs.size()) 1049 OutMIs.resize(NewInsnID + 1); 1050 1051 OutMIs[NewInsnID] = Builder.buildInstr(Opcode); 1052 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1053 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" 1054 << NewInsnID << "], " << Opcode << ")\n"); 1055 break; 1056 } 1057 1058 case GIR_BuildConstant: { 1059 uint64_t TempRegID = readULEB(); 1060 uint64_t Imm = readU64(); 1061 Builder.buildConstant(State.TempRegisters[TempRegID], Imm); 1062 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1063 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg[" 1064 << TempRegID << "], Imm=" << Imm << ")\n"); 1065 break; 1066 } 1067 1068 case GIR_RootToRootCopy: 1069 case GIR_Copy: { 1070 uint64_t NewInsnID = 1071 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB(); 1072 uint64_t OldInsnID = 1073 (MatcherOpcode == GIR_RootToRootCopy) ? 0 : readULEB(); 1074 uint64_t OpIdx = readULEB(); 1075 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1076 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); 1077 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1078 dbgs() 1079 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID 1080 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); 1081 break; 1082 } 1083 1084 case GIR_CopyOrAddZeroReg: { 1085 uint64_t NewInsnID = readULEB(); 1086 uint64_t OldInsnID = readULEB(); 1087 uint64_t OpIdx = readULEB(); 1088 uint16_t ZeroReg = readU16(); 1089 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1090 MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); 1091 if (isOperandImmEqual(MO, 0, MRI)) 1092 OutMIs[NewInsnID].addReg(ZeroReg); 1093 else 1094 OutMIs[NewInsnID].add(MO); 1095 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1096 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" 1097 << NewInsnID << "], MIs[" << OldInsnID << "], " 1098 << OpIdx << ", " << ZeroReg << ")\n"); 1099 break; 1100 } 1101 1102 case GIR_CopySubReg: { 1103 uint64_t NewInsnID = readULEB(); 1104 uint64_t OldInsnID = readULEB(); 1105 uint64_t OpIdx = readULEB(); 1106 uint16_t SubRegIdx = readU16(); 1107 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1108 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), 1109 0, SubRegIdx); 1110 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1111 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" 1112 << NewInsnID << "], MIs[" << OldInsnID << "], " 1113 << OpIdx << ", " << SubRegIdx << ")\n"); 1114 break; 1115 } 1116 1117 case GIR_AddImplicitDef: { 1118 uint64_t InsnID = readULEB(); 1119 uint16_t RegNum = readU16(); 1120 uint16_t Flags = readU16(); 1121 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1122 Flags |= RegState::Implicit; 1123 OutMIs[InsnID].addDef(RegNum, Flags); 1124 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1125 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" 1126 << InsnID << "], " << RegNum << ")\n"); 1127 break; 1128 } 1129 1130 case GIR_AddImplicitUse: { 1131 uint64_t InsnID = readULEB(); 1132 uint16_t RegNum = readU16(); 1133 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1134 OutMIs[InsnID].addUse(RegNum, RegState::Implicit); 1135 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1136 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" 1137 << InsnID << "], " << RegNum << ")\n"); 1138 break; 1139 } 1140 1141 case GIR_AddRegister: { 1142 uint64_t InsnID = readULEB(); 1143 uint16_t RegNum = readU16(); 1144 uint16_t RegFlags = readU16(); 1145 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1146 OutMIs[InsnID].addReg(RegNum, RegFlags); 1147 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1148 dbgs() 1149 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID 1150 << "], " << RegNum << ", " << RegFlags << ")\n"); 1151 break; 1152 } 1153 case GIR_AddIntrinsicID: { 1154 uint64_t InsnID = readULEB(); 1155 uint16_t Value = readU16(); 1156 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1157 OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value); 1158 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1159 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs[" 1160 << InsnID << "], " << Value << ")\n"); 1161 break; 1162 } 1163 case GIR_SetImplicitDefDead: { 1164 uint64_t InsnID = readULEB(); 1165 uint64_t OpIdx = readULEB(); 1166 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1167 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs[" 1168 << InsnID << "], OpIdx=" << OpIdx << ")\n"); 1169 MachineInstr *MI = OutMIs[InsnID]; 1170 assert(MI && "Modifying undefined instruction"); 1171 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead(); 1172 break; 1173 } 1174 case GIR_SetMIFlags: { 1175 uint64_t InsnID = readULEB(); 1176 uint32_t Flags = readU32(); 1177 1178 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1179 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs[" 1180 << InsnID << "], " << Flags << ")\n"); 1181 MachineInstr *MI = OutMIs[InsnID]; 1182 MI->setFlags(MI->getFlags() | Flags); 1183 break; 1184 } 1185 case GIR_UnsetMIFlags: { 1186 uint64_t InsnID = readULEB(); 1187 uint32_t Flags = readU32(); 1188 1189 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1190 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs[" 1191 << InsnID << "], " << Flags << ")\n"); 1192 MachineInstr *MI = OutMIs[InsnID]; 1193 MI->setFlags(MI->getFlags() & ~Flags); 1194 break; 1195 } 1196 case GIR_CopyMIFlags: { 1197 uint64_t InsnID = readULEB(); 1198 uint64_t OldInsnID = readULEB(); 1199 1200 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1201 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs[" 1202 << InsnID << "], MIs[" << OldInsnID << "])\n"); 1203 MachineInstr *MI = OutMIs[InsnID]; 1204 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags()); 1205 break; 1206 } 1207 case GIR_AddSimpleTempRegister: 1208 case GIR_AddTempRegister: 1209 case GIR_AddTempSubRegister: { 1210 uint64_t InsnID = readULEB(); 1211 uint64_t TempRegID = readULEB(); 1212 uint16_t TempRegFlags = 0; 1213 if (MatcherOpcode != GIR_AddSimpleTempRegister) 1214 TempRegFlags = readU16(); 1215 uint16_t SubReg = 0; 1216 if (MatcherOpcode == GIR_AddTempSubRegister) 1217 SubReg = readU16(); 1218 1219 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1220 1221 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags, 1222 SubReg); 1223 DEBUG_WITH_TYPE( 1224 TgtExecutor::getName(), 1225 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID 1226 << "], TempRegisters[" << TempRegID << "]"; 1227 if (SubReg) dbgs() << '.' << TRI.getSubRegIndexName(SubReg); 1228 dbgs() << ", " << TempRegFlags << ")\n"); 1229 break; 1230 } 1231 1232 case GIR_AddImm8: 1233 case GIR_AddImm: { 1234 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8); 1235 uint64_t InsnID = readULEB(); 1236 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64(); 1237 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1238 OutMIs[InsnID].addImm(Imm); 1239 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1240 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID 1241 << "], " << Imm << ")\n"); 1242 break; 1243 } 1244 1245 case GIR_AddCImm: { 1246 uint64_t InsnID = readULEB(); 1247 int TypeID = readS8(); 1248 uint64_t Imm = readU64(); 1249 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1250 1251 unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits(); 1252 LLVMContext &Ctx = MF->getFunction().getContext(); 1253 OutMIs[InsnID].addCImm( 1254 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, /*signed*/ true)); 1255 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1256 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID 1257 << "], TypeID=" << TypeID << ", Imm=" << Imm 1258 << ")\n"); 1259 break; 1260 } 1261 1262 case GIR_ComplexRenderer: { 1263 uint64_t InsnID = readULEB(); 1264 uint16_t RendererID = readU16(); 1265 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1266 for (const auto &RenderOpFn : State.Renderers[RendererID]) 1267 RenderOpFn(OutMIs[InsnID]); 1268 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1269 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" 1270 << InsnID << "], " << RendererID << ")\n"); 1271 break; 1272 } 1273 case GIR_ComplexSubOperandRenderer: { 1274 uint64_t InsnID = readULEB(); 1275 uint16_t RendererID = readU16(); 1276 uint64_t RenderOpID = readULEB(); 1277 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1278 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); 1279 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1280 dbgs() << CurrentIdx 1281 << ": GIR_ComplexSubOperandRenderer(OutMIs[" 1282 << InsnID << "], " << RendererID << ", " 1283 << RenderOpID << ")\n"); 1284 break; 1285 } 1286 case GIR_ComplexSubOperandSubRegRenderer: { 1287 uint64_t InsnID = readULEB(); 1288 uint16_t RendererID = readU16(); 1289 uint64_t RenderOpID = readULEB(); 1290 uint16_t SubRegIdx = readU16(); 1291 MachineInstrBuilder &MI = OutMIs[InsnID]; 1292 assert(MI && "Attempted to add to undefined instruction"); 1293 State.Renderers[RendererID][RenderOpID](MI); 1294 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx); 1295 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1296 dbgs() << CurrentIdx 1297 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs[" 1298 << InsnID << "], " << RendererID << ", " 1299 << RenderOpID << ", " << SubRegIdx << ")\n"); 1300 break; 1301 } 1302 1303 case GIR_CopyConstantAsSImm: { 1304 uint64_t NewInsnID = readULEB(); 1305 uint64_t OldInsnID = readULEB(); 1306 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1307 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && 1308 "Expected G_CONSTANT"); 1309 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { 1310 OutMIs[NewInsnID].addImm( 1311 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); 1312 } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) 1313 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); 1314 else 1315 llvm_unreachable("Expected Imm or CImm operand"); 1316 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1317 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" 1318 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1319 break; 1320 } 1321 1322 // TODO: Needs a test case once we have a pattern that uses this. 1323 case GIR_CopyFConstantAsFPImm: { 1324 uint64_t NewInsnID = readULEB(); 1325 uint64_t OldInsnID = readULEB(); 1326 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); 1327 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && 1328 "Expected G_FCONSTANT"); 1329 if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) 1330 OutMIs[NewInsnID].addFPImm( 1331 State.MIs[OldInsnID]->getOperand(1).getFPImm()); 1332 else 1333 llvm_unreachable("Expected FPImm operand"); 1334 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1335 dbgs() 1336 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" 1337 << NewInsnID << "], MIs[" << OldInsnID << "])\n"); 1338 break; 1339 } 1340 1341 case GIR_CustomRenderer: { 1342 uint64_t InsnID = readULEB(); 1343 uint64_t OldInsnID = readULEB(); 1344 uint16_t RendererFnID = readU16(); 1345 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1346 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1347 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" 1348 << InsnID << "], MIs[" << OldInsnID << "], " 1349 << RendererFnID << ")\n"); 1350 (Exec.*ExecInfo.CustomRenderers[RendererFnID])( 1351 OutMIs[InsnID], *State.MIs[OldInsnID], 1352 -1); // Not a source operand of the old instruction. 1353 break; 1354 } 1355 case GIR_DoneWithCustomAction: { 1356 uint16_t FnID = readU16(); 1357 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1358 dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID=" 1359 << FnID << ")\n"); 1360 assert(FnID > GICXXCustomAction_Invalid && "Expected a valid FnID"); 1361 if (runCustomAction(FnID, State, OutMIs)) { 1362 propagateFlags(); 1363 return true; 1364 } 1365 1366 if (handleReject() == RejectAndGiveUp) 1367 return false; 1368 break; 1369 } 1370 case GIR_CustomOperandRenderer: { 1371 uint64_t InsnID = readULEB(); 1372 uint64_t OldInsnID = readULEB(); 1373 uint64_t OpIdx = readULEB(); 1374 uint16_t RendererFnID = readU16(); 1375 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1376 1377 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1378 dbgs() << CurrentIdx 1379 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID 1380 << "], MIs[" << OldInsnID << "]->getOperand(" 1381 << OpIdx << "), " << RendererFnID << ")\n"); 1382 (Exec.*ExecInfo.CustomRenderers[RendererFnID])( 1383 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx); 1384 break; 1385 } 1386 case GIR_ConstrainOperandRC: { 1387 uint64_t InsnID = readULEB(); 1388 uint64_t OpIdx = readULEB(); 1389 uint16_t RCEnum = readU16(); 1390 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1391 MachineInstr &I = *OutMIs[InsnID].getInstr(); 1392 MachineFunction &MF = *I.getParent()->getParent(); 1393 MachineRegisterInfo &MRI = MF.getRegInfo(); 1394 const TargetRegisterClass &RC = *TRI.getRegClass(RCEnum); 1395 MachineOperand &MO = I.getOperand(OpIdx); 1396 constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, MO); 1397 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1398 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" 1399 << InsnID << "], " << OpIdx << ", " << RCEnum 1400 << ")\n"); 1401 break; 1402 } 1403 1404 case GIR_RootConstrainSelectedInstOperands: 1405 case GIR_ConstrainSelectedInstOperands: { 1406 uint64_t InsnID = (MatcherOpcode == GIR_RootConstrainSelectedInstOperands) 1407 ? 0 1408 : readULEB(); 1409 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1410 constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, 1411 RBI); 1412 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1413 dbgs() << CurrentIdx 1414 << ": GIR_ConstrainSelectedInstOperands(OutMIs[" 1415 << InsnID << "])\n"); 1416 break; 1417 } 1418 case GIR_MergeMemOperands: { 1419 uint64_t InsnID = readULEB(); 1420 uint64_t NumInsn = MatchTable[CurrentIdx++]; 1421 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); 1422 1423 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1424 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" 1425 << InsnID << "]"); 1426 for (unsigned K = 0; K < NumInsn; ++K) { 1427 uint64_t NextID = readULEB(); 1428 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1429 dbgs() << ", MIs[" << NextID << "]"); 1430 for (const auto &MMO : State.MIs[NextID]->memoperands()) 1431 OutMIs[InsnID].addMemOperand(MMO); 1432 } 1433 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n"); 1434 break; 1435 } 1436 case GIR_EraseFromParent: { 1437 uint64_t InsnID = readULEB(); 1438 MachineInstr *MI = State.MIs[InsnID]; 1439 assert(MI && "Attempted to erase an undefined instruction"); 1440 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1441 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" 1442 << InsnID << "])\n"); 1443 eraseImpl(MI); 1444 break; 1445 } 1446 case GIR_EraseRootFromParent_Done: { 1447 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1448 dbgs() 1449 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n"); 1450 eraseImpl(State.MIs[0]); 1451 propagateFlags(); 1452 return true; 1453 } 1454 case GIR_MakeTempReg: { 1455 uint64_t TempRegID = readULEB(); 1456 int TypeID = readS8(); 1457 1458 State.TempRegisters[TempRegID] = 1459 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID)); 1460 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1461 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID 1462 << "] = GIR_MakeTempReg(" << TypeID << ")\n"); 1463 break; 1464 } 1465 case GIR_ReplaceReg: { 1466 uint64_t OldInsnID = readULEB(); 1467 uint64_t OldOpIdx = readULEB(); 1468 uint64_t NewInsnID = readULEB(); 1469 uint64_t NewOpIdx = readULEB(); 1470 1471 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1472 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs[" 1473 << OldInsnID << "][" << OldOpIdx << "] = MIs[" 1474 << NewInsnID << "][" << NewOpIdx << "])\n"); 1475 1476 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); 1477 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg(); 1478 if (Observer) 1479 Observer->changingAllUsesOfReg(MRI, Old); 1480 MRI.replaceRegWith(Old, New); 1481 if (Observer) 1482 Observer->finishedChangingAllUsesOfReg(); 1483 break; 1484 } 1485 case GIR_ReplaceRegWithTempReg: { 1486 uint64_t OldInsnID = readULEB(); 1487 uint64_t OldOpIdx = readULEB(); 1488 uint64_t TempRegID = readULEB(); 1489 1490 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1491 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs[" 1492 << OldInsnID << "][" << OldOpIdx << "] = TempRegs[" 1493 << TempRegID << "])\n"); 1494 1495 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg(); 1496 Register New = State.TempRegisters[TempRegID]; 1497 if (Observer) 1498 Observer->changingAllUsesOfReg(MRI, Old); 1499 MRI.replaceRegWith(Old, New); 1500 if (Observer) 1501 Observer->finishedChangingAllUsesOfReg(); 1502 break; 1503 } 1504 case GIR_Coverage: { 1505 uint32_t RuleID = readU32(); 1506 assert(CoverageInfo); 1507 CoverageInfo->setCovered(RuleID); 1508 1509 DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << CurrentIdx 1510 << ": GIR_Coverage(" 1511 << RuleID << ")"); 1512 break; 1513 } 1514 1515 case GIR_Done: 1516 DEBUG_WITH_TYPE(TgtExecutor::getName(), 1517 dbgs() << CurrentIdx << ": GIR_Done\n"); 1518 propagateFlags(); 1519 return true; 1520 default: 1521 llvm_unreachable("Unexpected command"); 1522 } 1523 } 1524 } 1525 1526 } // end namespace llvm 1527 1528 #endif // LLVM_CODEGEN_GLOBALISEL_GIMATCHTABLEEXECUTORIMPL_H 1529