Lines Matching full:i
93 bool select(MachineInstr &I) override;
107 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
112 MachineInstr &I) const;
114 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
118 MachineInstr &I, Register SrcReg,
120 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
124 MachineInstr &I) const;
127 MachineInstr &I) const;
128 bool selectStore(MachineInstr &I) const;
131 MachineInstr &I) const;
132 bool selectStackRestore(MachineInstr &I) const;
134 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
137 MachineInstr &I, unsigned NewOpcode,
141 MachineInstr &I) const;
143 bool selectFence(MachineInstr &I) const;
146 MachineInstr &I) const;
149 MachineInstr &I, unsigned OpType) const;
152 MachineInstr &I) const;
155 MachineInstr &I) const;
158 MachineInstr &I) const;
161 MachineInstr &I) const;
163 MachineInstr &I) const;
166 unsigned comparisonOpcode, MachineInstr &I) const;
169 MachineInstr &I) const;
171 MachineInstr &I) const;
174 MachineInstr &I) const;
177 MachineInstr &I) const;
179 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
181 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
185 MachineInstr &I) const;
187 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
189 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
191 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
195 MachineInstr &I) const;
197 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
201 MachineInstr &I) const;
203 MachineInstr &I) const;
205 MachineInstr &I) const;
207 MachineInstr &I) const;
209 MachineInstr &I) const;
211 MachineInstr &I) const;
213 MachineInstr &I) const;
215 MachineInstr &I) const;
218 MachineInstr &I) const;
220 MachineInstr &I) const;
222 bool selectBranch(MachineInstr &I) const;
223 bool selectBranchCond(MachineInstr &I) const;
226 MachineInstr &I) const;
229 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
231 MachineInstr &I, CL::OpenCLExtInst CLInst,
234 MachineInstr &I, const ExtInstList &ExtInsts) const;
237 MachineInstr &I) const;
240 MachineInstr &I) const;
242 bool selectUnmergeValues(MachineInstr &I) const;
244 Register buildI32Constant(uint32_t Val, MachineInstr &I,
247 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
248 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const;
250 MachineInstr &I) const;
252 bool wrapIntoSpecConstantOp(MachineInstr &I,
291 bool SPIRVInstructionSelector::select(MachineInstr &I) {
292 assert(I.getParent() && "Instruction should be in a basic block!");
293 assert(I.getParent()->getParent() && "Instruction should be in a function!");
295 Register Opcode = I.getOpcode();
299 Register DstReg = I.getOperand(0).getReg();
300 Register SrcReg = I.getOperand(1).getReg();
305 bool Res = selectImpl(I, *CoverageInfo);
311 I.removeFromParent();
313 } else if (I.getNumDefs() == 1) {
315 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32));
317 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
320 if (I.getNumOperands() != I.getNumExplicitOperands()) {
327 bool HasDefs = I.getNumDefs() > 0;
328 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
330 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
331 if (spvSelect(ResVReg, ResType, I)) {
333 for (unsigned i = 0; i < I.getNumDefs(); ++i)
334 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32));
335 I.removeFromParent();
343 MachineInstr &I) const {
344 const unsigned Opcode = I.getOpcode();
346 return selectImpl(I, *CoverageInfo);
349 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(),
350 I);
352 return selectGlobalValue(ResVReg, I);
354 return selectOpUndef(ResVReg, ResType, I);
356 return selectFreeze(ResVReg, ResType, I);
361 return selectIntrinsic(ResVReg, ResType, I);
363 return selectBitreverse(ResVReg, ResType, I);
366 return selectConstVector(ResVReg, ResType, I);
368 return selectSplatVector(ResVReg, ResType, I);
371 MachineBasicBlock &BB = *I.getParent();
372 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
375 .addUse(I.getOperand(1).getReg())
376 .addUse(I.getOperand(2).getReg());
377 for (auto V : I.getOperand(3).getShuffleMask())
384 return selectMemOperation(ResVReg, I);
387 return selectICmp(ResVReg, ResType, I);
389 return selectFCmp(ResVReg, ResType, I);
392 return selectFrameIndex(ResVReg, ResType, I);
395 return selectLoad(ResVReg, ResType, I);
397 return selectStore(I);
400 return selectBranch(I);
402 return selectBranchCond(I);
405 return selectPhi(ResVReg, ResType, I);
408 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
410 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
413 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
415 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
418 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
420 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
422 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
425 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
427 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
430 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
433 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
435 return selectExtInst(ResVReg, ResType, I, CL::pown);
438 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
440 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
443 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
445 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
447 return selectLog10(ResVReg, ResType, I);
450 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
452 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
456 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
459 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
462 return selectExtInst(ResVReg, ResType, I, CL::copysign);
465 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
467 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
470 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
472 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
474 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
476 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos);
478 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin);
480 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan);
482 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh);
484 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh);
486 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh);
489 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
493 return selectExtInst(ResVReg, ResType, I, CL::ctz);
496 return selectExtInst(ResVReg, ResType, I, CL::clz);
499 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
501 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
503 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
506 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
509 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
511 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
514 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat);
516 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat);
518 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat);
520 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
523 return selectExt(ResVReg, ResType, I, true);
526 return selectExt(ResVReg, ResType, I, false);
528 return selectTrunc(ResVReg, ResType, I);
531 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
534 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
536 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
538 return selectBitcast(ResVReg, ResType, I);
540 return selectAddrSpaceCast(ResVReg, ResType, I);
546 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
547 Register GV = I.getOperand(1).getReg();
553 isImm(I.getOperand(2), MRI));
554 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
555 MachineBasicBlock &BB = *I.getParent();
556 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
563 .addUse(I.getOperand(2).getReg());
568 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
570 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
572 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
574 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
576 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
578 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
580 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
582 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
584 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
586 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
588 return selectAtomicCmpXchg(ResVReg, ResType, I);
591 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
594 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
597 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
599 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
602 return selectFence(I);
605 return selectStackSave(ResVReg, ResType, I);
607 return selectStackRestore(I);
610 return selectUnmergeValues(I);
619 MachineInstr &I,
621 return selectExtInst(ResVReg, ResType, I,
627 MachineInstr &I,
632 return selectExtInst(ResVReg, ResType, I, ExtInsts);
637 MachineInstr &I,
644 MachineBasicBlock &BB = *I.getParent();
645 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
650 const unsigned NumOps = I.getNumOperands();
651 for (unsigned i = 1; i < NumOps; ++i)
652 MIB.add(I.getOperand(i));
661 MachineInstr &I,
664 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
673 MachineInstr &I,
675 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) {
676 Register SrcReg = I.getOperand(1).getReg();
697 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
706 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(),
712 MachineInstr &I) const {
713 Register OpReg = I.getOperand(1).getReg();
717 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
773 MachineInstr &I) const {
774 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
775 Register Ptr = I.getOperand(1 + OpOffset).getReg();
776 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
780 if (!I.getNumMemOperands()) {
781 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
782 I.getOpcode() ==
784 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
786 addMemoryOperands(*I.memoperands_begin(), MIB);
791 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
792 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
793 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
794 Register Ptr = I.getOperand(1 + OpOffset).getReg();
795 MachineBasicBlock &BB = *I.getParent();
796 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
799 if (!I.getNumMemOperands()) {
800 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
801 I.getOpcode() ==
803 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
805 addMemoryOperands(*I.memoperands_begin(), MIB);
812 MachineInstr &I) const {
818 MachineBasicBlock &BB = *I.getParent();
819 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
825 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
831 if (!I.getOperand(0).isReg())
833 MachineBasicBlock &BB = *I.getParent();
834 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
835 .addUse(I.getOperand(0).getReg())
840 MachineInstr &I) const {
841 MachineBasicBlock &BB = *I.getParent();
842 Register SrcReg = I.getOperand(1).getReg();
843 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
844 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
845 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
846 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
847 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
848 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
849 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII);
851 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
863 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
864 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
871 ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
873 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast);
875 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
876 .addUse(I.getOperand(0).getReg())
878 .addUse(I.getOperand(2).getReg());
879 if (I.getNumMemOperands())
880 addMemoryOperands(*I.memoperands_begin(), MIB);
883 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg)
890 MachineInstr &I,
893 assert(I.hasOneMemOperand());
894 const MachineMemOperand *MemOp = *I.memoperands_begin();
897 Register ScopeReg = buildI32Constant(Scope, I);
899 Register Ptr = I.getOperand(1).getReg();
905 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I);
908 Register ValueReg = I.getOperand(2).getReg();
912 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode);
916 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
927 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
928 unsigned ArgI = I.getNumOperands() - 1;
930 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
939 MachineBasicBlock &BB = *I.getParent();
941 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
942 Register ResVReg = I.getOperand(i).getReg();
952 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
956 .addImm(static_cast<int64_t>(i));
962 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
963 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
965 Register MemSemReg = buildI32Constant(MemSem, I);
966 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
968 Register ScopeReg = buildI32Constant(Scope, I);
969 MachineBasicBlock &BB = *I.getParent();
970 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
978 MachineInstr &I) const {
982 Register Ptr = I.getOperand(2).getReg();
983 if (!isa<GIntrinsic>(I)) {
984 assert(I.hasOneMemOperand());
985 const MachineMemOperand *MemOp = *I.memoperands_begin();
988 ScopeReg = buildI32Constant(Scope, I);
994 MemSemEqReg = buildI32Constant(MemSemEq, I);
998 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I);
1000 ScopeReg = I.getOperand(5).getReg();
1001 MemSemEqReg = I.getOperand(6).getReg();
1002 MemSemNeqReg = I.getOperand(7).getReg();
1005 Register Cmp = I.getOperand(3).getReg();
1006 Register Val = I.getOperand(4).getReg();
1009 const DebugLoc &DL = I.getDebugLoc();
1011 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
1022 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
1023 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
1030 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
1034 .addUse(GR.getOrCreateUndef(I, ResType, TII))
1037 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
1075 MachineInstr &I) const {
1083 Register NewReg = I.getOperand(1).getReg();
1084 MachineBasicBlock &BB = *I.getParent();
1085 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
1086 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII,
1089 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1097 Register SrcPtr = I.getOperand(1).getReg();
1104 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1112 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
1115 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
1120 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic);
1121 MachineBasicBlock &BB = *I.getParent();
1122 const DebugLoc &DL = I.getDebugLoc();
1123 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
1128 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
1138 return selectUnOp(ResVReg, ResType, I,
1141 return selectUnOp(ResVReg, ResType, I,
1144 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
1146 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
1242 MachineInstr &I,
1244 assert(I.getNumOperands() == 3);
1245 assert(I.getOperand(2).isReg());
1246 MachineBasicBlock &BB = *I.getParent();
1247 Register InputRegister = I.getOperand(2).getReg();
1256 assert(ResVReg == I.getOperand(0).getReg());
1257 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1267 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
1275 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
1280 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
1282 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
1293 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
1302 MachineInstr &I) const {
1303 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
1308 MachineInstr &I) const {
1309 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
1314 MachineInstr &I) const {
1316 assert(I.getNumOperands() == 5);
1317 assert(I.getOperand(2).isReg());
1318 assert(I.getOperand(3).isReg());
1319 assert(I.getOperand(4).isReg());
1320 MachineBasicBlock &BB = *I.getParent();
1322 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1327 .addUse(I.getOperand(2).getReg())
1328 .addUse(I.getOperand(3).getReg())
1329 .addUse(I.getOperand(4).getReg())
1335 MachineInstr &I) const {
1337 assert(I.getNumOperands() == 3);
1338 assert(I.getOperand(2).isReg());
1339 MachineBasicBlock &BB = *I.getParent();
1341 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1346 .addUse(I.getOperand(2).getReg())
1352 MachineInstr &I) const {
1353 MachineBasicBlock &BB = *I.getParent();
1354 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
1357 .addUse(I.getOperand(1).getReg())
1363 MachineInstr &I) const {
1369 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg())
1371 Register OpReg = I.getOperand(1).getReg();
1393 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
1394 .addDef(I.getOperand(0).getReg())
1403 MachineInstr &I) const {
1406 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) {
1421 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1425 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
1426 MIB.addUse(I.getOperand(i).getReg());
1472 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands();
1473 i++) {
1475 OpDef->getOperand(i).isReg()
1476 ? MRI->getVRegDef(OpDef->getOperand(i).getReg())
1497 MachineInstr &I) const {
1506 unsigned OpIdx = I.getNumExplicitDefs();
1507 if (!I.getOperand(OpIdx).isReg())
1511 Register OpReg = I.getOperand(OpIdx).getReg();
1518 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1523 for (unsigned i = 0; i < N; ++i)
1531 MachineInstr &I) const {
1532 Register Cmp0 = I.getOperand(2).getReg();
1533 Register Cmp1 = I.getOperand(3).getReg();
1537 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
1547 MachineInstr &I) const {
1548 auto Pred = I.getOperand(1).getPredicate();
1551 Register CmpOperand = I.getOperand(2).getReg();
1558 return selectCmp(ResVReg, ResType, CmpOpc, I);
1562 const MachineInstr &I,
1564 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
1566 const ConstantFP *FPImm = I.getOperand(1).getFPImm();
1571 const MachineInstr &I,
1573 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1575 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB);
1579 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
1583 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
1591 MachineBasicBlock &BB = *I.getParent();
1593 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1597 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1609 MachineInstr &I) const {
1610 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
1611 return selectCmp(ResVReg, ResType, CmpOp, I);
1615 MachineInstr &I) const {
1619 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
1620 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
1638 MachineInstr &I) const {
1643 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
1644 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
1649 MachineInstr &I) const {
1654 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII);
1655 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII);
1660 MachineInstr &I,
1663 Register ZeroReg = buildZerosVal(ResType, I);
1664 Register OneReg = buildOnesVal(IsSigned, ResType, I);
1666 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
1669 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1672 .addUse(I.getOperand(1).getReg())
1680 MachineInstr &I, bool IsSigned,
1682 Register SrcReg = I.getOperand(1).getReg();
1685 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
1687 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1690 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
1693 selectSelect(SrcReg, TmpType, I, false);
1695 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode);
1700 MachineInstr &I, bool IsSigned) const {
1701 Register SrcReg = I.getOperand(1).getReg();
1703 return selectSelect(ResVReg, ResType, I, IsSigned);
1707 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1714 return selectUnOp(ResVReg, ResType, I, Opcode);
1719 MachineInstr &I,
1726 Register Zero = buildZerosVal(IntTy, I);
1727 Register One = buildOnesVal(false, IntTy, I);
1728 MachineBasicBlock &BB = *I.getParent();
1729 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1735 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
1745 MachineInstr &I) const {
1746 Register IntReg = I.getOperand(1).getReg();
1749 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
1751 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1758 return selectUnOp(ResVReg, ResType, I, Opcode);
1764 MachineInstr &I) const {
1767 MachineBasicBlock &BB = *I.getParent();
1770 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1776 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
1779 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1784 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1795 MachineInstr &I) const {
1796 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1824 MachineInstr &I) const {
1825 MachineBasicBlock &BB = *I.getParent();
1826 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
1830 .addUse(I.getOperand(3).getReg())
1832 .addUse(I.getOperand(2).getReg());
1833 for (unsigned i = 4; i < I.getNumOperands(); i++)
1834 MIB.addImm(foldImm(I.getOperand(i), MRI));
1840 MachineInstr &I) const {
1841 MachineBasicBlock &BB = *I.getParent();
1842 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1845 .addUse(I.getOperand(2).getReg());
1846 for (unsigned i = 3; i < I.getNumOperands(); i++)
1847 MIB.addImm(foldImm(I.getOperand(i), MRI));
1853 MachineInstr &I) const {
1854 if (isImm(I.getOperand(4), MRI))
1855 return selectInsertVal(ResVReg, ResType, I);
1856 MachineBasicBlock &BB = *I.getParent();
1857 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
1860 .addUse(I.getOperand(2).getReg())
1861 .addUse(I.getOperand(3).getReg())
1862 .addUse(I.getOperand(4).getReg())
1868 MachineInstr &I) const {
1869 if (isImm(I.getOperand(3), MRI))
1870 return selectExtractVal(ResVReg, ResType, I);
1871 MachineBasicBlock &BB = *I.getParent();
1872 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
1875 .addUse(I.getOperand(2).getReg())
1876 .addUse(I.getOperand(3).getReg())
1882 MachineInstr &I) const {
1883 const bool IsGEPInBounds = I.getOperand(2).getImm();
1894 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1898 .addUse(I.getOperand(3).getReg());
1904 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
1905 Res.addUse(I.getOperand(i).getReg());
1911 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
1913 unsigned Lim = I.getNumExplicitOperands();
1914 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
1915 Register OpReg = I.getOperand(i).getReg();
1927 MachineFunction *MF = I.getMF();
1940 MachineBasicBlock &BB = *I.getParent();
1941 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1955 MachineInstr &I) const {
1956 MachineBasicBlock &BB = *I.getParent();
1957 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
1960 return selectLoad(ResVReg, ResType, I);
1962 return selectStore(I);
1964 return selectExtractVal(ResVReg, ResType, I);
1966 return selectInsertVal(ResVReg, ResType, I);
1968 return selectExtractElt(ResVReg, ResType, I);
1970 return selectInsertElt(ResVReg, ResType, I);
1972 return selectGEP(ResVReg, ResType, I);
1975 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
1976 MachineInstr *Init = I.getNumExplicitOperands() > 2
1977 ? MRI->getVRegDef(I.getOperand(2).getReg())
1983 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1990 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
1996 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
1999 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2010 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
2011 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
2012 for (unsigned i = I.getNumExplicitDefs() + 2;
2013 i < I.getNumExplicitOperands(); ++i) {
2014 MIB.addImm(I.getOperand(i).getImm());
2019 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
2020 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
2021 if (I.getOperand(i).isReg())
2022 MIB.addReg(I.getOperand(i).getReg());
2023 else if (I.getOperand(i).isCImm())
2024 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
2025 else if (I.getOperand(i).isMBB())
2026 MIB.addMBB(I.getOperand(i).getMBB());
2033 return selectAtomicCmpXchg(ResVReg, ResType, I);
2035 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable));
2038 return selectFrameIndex(ResVReg, ResType, I);
2040 return selectAllocaArray(ResVReg, ResType, I);
2043 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
2044 .addUse(I.getOperand(1).getReg());
2048 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
2051 .addUse(I.getOperand(2).getReg())
2052 .addUse(I.getOperand(3).getReg());
2055 return selectSpvThreadId(ResVReg, ResType, I);
2057 return selectAll(ResVReg, ResType, I);
2059 return selectAny(ResVReg, ResType, I);
2061 return selectFmix(ResVReg, ResType, I);
2063 return selectRsqrt(ResVReg, ResType, I);
2068 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
2069 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
2074 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size);
2079 I.print(OS);
2089 MachineInstr &I) const {
2092 MachineBasicBlock &BB = *I.getParent();
2093 return BuildMI(BB, I, I.getDebugLoc(),
2097 .addUse(I.getOperand(2).getReg())
2103 MachineInstr &I) const {
2106 MachineFunction *MF = I.getParent()->getParent();
2111 for (; It != E && It != I; ++It) {
2128 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
2133 const MachineInstr *PrevI = I.getPrevNode();
2134 MachineBasicBlock &MBB = *I.getParent();
2136 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
2139 .addMBB(I.getOperand(0).getMBB())
2142 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
2143 .addMBB(I.getOperand(0).getMBB())
2147 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
2158 const MachineInstr *NextI = I.getNextNode();
2164 MachineBasicBlock &MBB = *I.getParent();
2166 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
2167 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
2168 .addUse(I.getOperand(0).getReg())
2169 .addMBB(I.getOperand(1).getMBB())
2176 MachineInstr &I) const {
2177 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi))
2180 const unsigned NumOps = I.getNumOperands();
2181 for (unsigned i = 1; i < NumOps; i += 2) {
2182 MIB.addUse(I.getOperand(i + 0).getReg());
2183 MIB.addMBB(I.getOperand(i + 1).getMBB());
2189 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
2191 MachineIRBuilder MIRBuilder(I);
2192 const GlobalValue *GV = I.getOperand(1).getGlobal();
2200 ArrayElementType, GVType->getArrayNumElements(), I, TII);
2206 PointerBaseType, I, TII,
2231 MachineBasicBlock &BB = *I.getParent();
2248 BuildMI(BB, I, I.getDebugLoc(),
2257 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
2263 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
2299 MachineInstr &I) const {
2301 return selectExtInst(ResVReg, ResType, I, CL::log10);
2309 MachineIRBuilder MIRBuilder(I);
2310 MachineBasicBlock &BB = *I.getParent();
2315 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
2320 .add(I.getOperand(1))
2338 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2350 MachineInstr &I) const {
2355 MachineIRBuilder MIRBuilder(I);
2382 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
2389 assert(I.getOperand(2).isReg());
2390 Register ThreadIdReg = I.getOperand(2).getReg();
2401 MachineBasicBlock &BB = *I.getParent();
2402 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))