1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 #include "llvm/Support/ErrorHandling.h" 37 38 #define DEBUG_TYPE "spirv-isel" 39 40 using namespace llvm; 41 namespace CL = SPIRV::OpenCLExtInst; 42 namespace GL = SPIRV::GLSLExtInst; 43 44 using ExtInstList = 45 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 46 47 namespace { 48 49 llvm::SPIRV::SelectionControl::SelectionControl 50 getSelectionOperandForImm(int Imm) { 51 if (Imm == 2) 52 return SPIRV::SelectionControl::Flatten; 53 if (Imm == 1) 54 return SPIRV::SelectionControl::DontFlatten; 55 if (Imm == 0) 56 return SPIRV::SelectionControl::None; 57 llvm_unreachable("Invalid immediate"); 58 } 59 60 #define GET_GLOBALISEL_PREDICATE_BITSET 61 #include "SPIRVGenGlobalISel.inc" 62 #undef GET_GLOBALISEL_PREDICATE_BITSET 63 64 class SPIRVInstructionSelector : public InstructionSelector { 65 const SPIRVSubtarget &STI; 66 const SPIRVInstrInfo &TII; 67 const SPIRVRegisterInfo &TRI; 68 const RegisterBankInfo &RBI; 69 SPIRVGlobalRegistry &GR; 70 MachineRegisterInfo *MRI; 71 MachineFunction *HasVRegsReset = nullptr; 72 73 /// We need to keep track of the number we give to anonymous global values to 74 /// generate the same name every time when this is needed. 75 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 76 SmallPtrSet<MachineInstr *, 8> DeadMIs; 77 78 public: 79 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 80 const SPIRVSubtarget &ST, 81 const RegisterBankInfo &RBI); 82 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 83 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 84 BlockFrequencyInfo *BFI) override; 85 // Common selection code. Instruction-specific selection occurs in spvSelect. 86 bool select(MachineInstr &I) override; 87 static const char *getName() { return DEBUG_TYPE; } 88 89 #define GET_GLOBALISEL_PREDICATES_DECL 90 #include "SPIRVGenGlobalISel.inc" 91 #undef GET_GLOBALISEL_PREDICATES_DECL 92 93 #define GET_GLOBALISEL_TEMPORARIES_DECL 94 #include "SPIRVGenGlobalISel.inc" 95 #undef GET_GLOBALISEL_TEMPORARIES_DECL 96 97 private: 98 void resetVRegsType(MachineFunction &MF); 99 100 // tblgen-erated 'select' implementation, used as the initial selector for 101 // the patterns that don't require complex C++. 102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 103 104 // All instruction-specific selection that didn't happen in "select()". 105 // Is basically a large Switch/Case delegating to all other select method. 106 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I) const; 108 109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 110 MachineInstr &I, bool IsSigned) const; 111 112 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I, bool IsSigned) const; 114 115 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, Register SrcReg, 117 bool IsSigned) const; 118 119 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I, bool IsSigned) const; 121 122 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 123 const MachineInstr *Init = nullptr) const; 124 125 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 126 MachineInstr &I, std::vector<Register> SrcRegs, 127 unsigned Opcode) const; 128 129 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 130 unsigned Opcode) const; 131 132 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I) const; 134 135 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I) const; 137 bool selectStore(MachineInstr &I) const; 138 139 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 140 MachineInstr &I) const; 141 bool selectStackRestore(MachineInstr &I) const; 142 143 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 144 145 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I, unsigned NewOpcode, 147 unsigned NegateOpcode = 0) const; 148 149 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 152 bool selectFence(MachineInstr &I) const; 153 154 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 155 MachineInstr &I) const; 156 157 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 158 MachineInstr &I, unsigned OpType) const; 159 160 bool selectAll(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 163 bool selectAny(Register ResVReg, const SPIRVType *ResType, 164 MachineInstr &I) const; 165 166 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 167 MachineInstr &I) const; 168 169 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I) const; 171 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 172 MachineInstr &I) const; 173 174 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 175 unsigned comparisonOpcode, MachineInstr &I) const; 176 bool selectCross(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 179 MachineInstr &I) const; 180 181 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 184 MachineInstr &I) const; 185 186 bool selectSign(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 189 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 190 MachineInstr &I) const; 191 192 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I, unsigned Opcode) const; 194 195 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 196 MachineInstr &I, bool Signed) const; 197 198 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 199 MachineInstr &I) const; 200 201 template <bool Signed> 202 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I) const; 204 template <bool Signed> 205 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I) const; 207 208 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 209 int OpIdx) const; 210 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 211 int OpIdx) const; 212 213 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 214 MachineInstr &I) const; 215 216 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 217 bool IsSigned) const; 218 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 219 bool IsSigned, unsigned Opcode) const; 220 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 221 bool IsSigned) const; 222 223 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 224 MachineInstr &I) const; 225 226 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 227 bool IsSigned) const; 228 229 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 230 const SPIRVType *intTy, const SPIRVType *boolTy) const; 231 232 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 233 MachineInstr &I) const; 234 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 235 MachineInstr &I) const; 236 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 237 MachineInstr &I) const; 238 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 239 MachineInstr &I) const; 240 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 241 MachineInstr &I) const; 242 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 243 MachineInstr &I) const; 244 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 245 MachineInstr &I) const; 246 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 247 MachineInstr &I) const; 248 249 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 250 MachineInstr &I) const; 251 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I) const; 253 254 bool selectBranch(MachineInstr &I) const; 255 bool selectBranchCond(MachineInstr &I) const; 256 257 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I) const; 259 260 [[maybe_unused]] bool selectExtInst(Register ResVReg, 261 const SPIRVType *RestType, 262 MachineInstr &I, 263 GL::GLSLExtInst GLInst) const; 264 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 265 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 266 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 267 MachineInstr &I, CL::OpenCLExtInst CLInst, 268 GL::GLSLExtInst GLInst) const; 269 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 270 MachineInstr &I, const ExtInstList &ExtInsts) const; 271 272 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 273 MachineInstr &I) const; 274 275 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 276 MachineInstr &I) const; 277 278 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 279 MachineInstr &I, unsigned Opcode) const; 280 281 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 282 MachineInstr &I) const; 283 284 bool selectUnmergeValues(MachineInstr &I) const; 285 286 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 287 MachineInstr &I) const; 288 289 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 290 MachineInstr &I) const; 291 292 bool selectImageWriteIntrinsic(MachineInstr &I) const; 293 294 // Utilities 295 std::pair<Register, bool> 296 buildI32Constant(uint32_t Val, MachineInstr &I, 297 const SPIRVType *ResType = nullptr) const; 298 299 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 300 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 301 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 302 MachineInstr &I) const; 303 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 304 305 bool wrapIntoSpecConstantOp(MachineInstr &I, 306 SmallVector<Register> &CompositeArgs) const; 307 308 Register getUcharPtrTypeReg(MachineInstr &I, 309 SPIRV::StorageClass::StorageClass SC) const; 310 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 311 Register Src, Register DestType, 312 uint32_t Opcode) const; 313 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 314 SPIRVType *SrcPtrTy) const; 315 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 316 uint32_t Binding, uint32_t ArraySize, 317 Register IndexReg, bool IsNonUniform, 318 MachineIRBuilder MIRBuilder) const; 319 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 320 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType, 321 Register &ReadReg, MachineInstr &InsertionPoint) const; 322 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 323 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 324 Register ResVReg, const SPIRVType *ResType, 325 MachineInstr &I) const; 326 }; 327 328 } // end anonymous namespace 329 330 #define GET_GLOBALISEL_IMPL 331 #include "SPIRVGenGlobalISel.inc" 332 #undef GET_GLOBALISEL_IMPL 333 334 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 335 const SPIRVSubtarget &ST, 336 const RegisterBankInfo &RBI) 337 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 338 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 339 #define GET_GLOBALISEL_PREDICATES_INIT 340 #include "SPIRVGenGlobalISel.inc" 341 #undef GET_GLOBALISEL_PREDICATES_INIT 342 #define GET_GLOBALISEL_TEMPORARIES_INIT 343 #include "SPIRVGenGlobalISel.inc" 344 #undef GET_GLOBALISEL_TEMPORARIES_INIT 345 { 346 } 347 348 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 349 CodeGenCoverage *CoverageInfo, 350 ProfileSummaryInfo *PSI, 351 BlockFrequencyInfo *BFI) { 352 MRI = &MF.getRegInfo(); 353 GR.setCurrentFunc(MF); 354 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 355 } 356 357 // Ensure that register classes correspond to pattern matching rules. 358 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 359 if (HasVRegsReset == &MF) 360 return; 361 HasVRegsReset = &MF; 362 363 MachineRegisterInfo &MRI = MF.getRegInfo(); 364 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 365 Register Reg = Register::index2VirtReg(I); 366 LLT RegType = MRI.getType(Reg); 367 if (RegType.isScalar()) 368 MRI.setType(Reg, LLT::scalar(64)); 369 else if (RegType.isPointer()) 370 MRI.setType(Reg, LLT::pointer(0, 64)); 371 else if (RegType.isVector()) 372 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 373 } 374 for (const auto &MBB : MF) { 375 for (const auto &MI : MBB) { 376 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 377 continue; 378 Register DstReg = MI.getOperand(0).getReg(); 379 LLT DstType = MRI.getType(DstReg); 380 Register SrcReg = MI.getOperand(1).getReg(); 381 LLT SrcType = MRI.getType(SrcReg); 382 if (DstType != SrcType) 383 MRI.setType(DstReg, MRI.getType(SrcReg)); 384 385 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 386 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 387 if (DstRC != SrcRC && SrcRC) 388 MRI.setRegClass(DstReg, SrcRC); 389 } 390 } 391 } 392 393 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 394 395 // Defined in SPIRVLegalizerInfo.cpp. 396 extern bool isTypeFoldingSupported(unsigned Opcode); 397 398 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 399 for (const auto &MO : MI.all_defs()) { 400 Register Reg = MO.getReg(); 401 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 402 return false; 403 } 404 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 405 MI.isLifetimeMarker()) 406 return false; 407 if (MI.isPHI()) 408 return true; 409 if (MI.mayStore() || MI.isCall() || 410 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 411 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 412 return false; 413 return true; 414 } 415 416 bool SPIRVInstructionSelector::select(MachineInstr &I) { 417 resetVRegsType(*I.getParent()->getParent()); 418 419 assert(I.getParent() && "Instruction should be in a basic block!"); 420 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 421 422 Register Opcode = I.getOpcode(); 423 // If it's not a GMIR instruction, we've selected it already. 424 if (!isPreISelGenericOpcode(Opcode)) { 425 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 426 Register DstReg = I.getOperand(0).getReg(); 427 Register SrcReg = I.getOperand(1).getReg(); 428 auto *Def = MRI->getVRegDef(SrcReg); 429 if (isTypeFoldingSupported(Def->getOpcode())) { 430 bool Res = selectImpl(I, *CoverageInfo); 431 LLVM_DEBUG({ 432 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 433 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 434 I.print(dbgs()); 435 } 436 }); 437 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 438 if (Res) { 439 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 440 DeadMIs.insert(Def); 441 return Res; 442 } 443 } 444 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 445 MRI->replaceRegWith(SrcReg, DstReg); 446 GR.invalidateMachineInstr(&I); 447 I.removeFromParent(); 448 return true; 449 } else if (I.getNumDefs() == 1) { 450 // Make all vregs 64 bits (for SPIR-V IDs). 451 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 452 } 453 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 454 } 455 456 if (DeadMIs.contains(&I)) { 457 // if the instruction has been already made dead by folding it away 458 // erase it 459 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 460 salvageDebugInfo(*MRI, I); 461 GR.invalidateMachineInstr(&I); 462 I.eraseFromParent(); 463 return true; 464 } 465 466 if (I.getNumOperands() != I.getNumExplicitOperands()) { 467 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 468 return false; 469 } 470 471 // Common code for getting return reg+type, and removing selected instr 472 // from parent occurs here. Instr-specific selection happens in spvSelect(). 473 bool HasDefs = I.getNumDefs() > 0; 474 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 475 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 476 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 477 if (spvSelect(ResVReg, ResType, I)) { 478 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 479 for (unsigned i = 0; i < I.getNumDefs(); ++i) 480 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 481 GR.invalidateMachineInstr(&I); 482 I.removeFromParent(); 483 return true; 484 } 485 return false; 486 } 487 488 static bool mayApplyGenericSelection(unsigned Opcode) { 489 switch (Opcode) { 490 case TargetOpcode::G_CONSTANT: 491 return false; 492 case TargetOpcode::G_SADDO: 493 case TargetOpcode::G_SSUBO: 494 return true; 495 } 496 return isTypeFoldingSupported(Opcode); 497 } 498 499 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 500 MachineInstr &I) const { 501 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 502 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 503 if (DstRC != SrcRC && SrcRC) 504 MRI->setRegClass(DestReg, SrcRC); 505 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 506 TII.get(TargetOpcode::COPY)) 507 .addDef(DestReg) 508 .addUse(SrcReg) 509 .constrainAllUses(TII, TRI, RBI); 510 } 511 512 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 513 const SPIRVType *ResType, 514 MachineInstr &I) const { 515 const unsigned Opcode = I.getOpcode(); 516 if (mayApplyGenericSelection(Opcode)) 517 return selectImpl(I, *CoverageInfo); 518 switch (Opcode) { 519 case TargetOpcode::G_CONSTANT: 520 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 521 I); 522 case TargetOpcode::G_GLOBAL_VALUE: 523 return selectGlobalValue(ResVReg, I); 524 case TargetOpcode::G_IMPLICIT_DEF: 525 return selectOpUndef(ResVReg, ResType, I); 526 case TargetOpcode::G_FREEZE: 527 return selectFreeze(ResVReg, ResType, I); 528 529 case TargetOpcode::G_INTRINSIC: 530 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 531 case TargetOpcode::G_INTRINSIC_CONVERGENT: 532 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 533 return selectIntrinsic(ResVReg, ResType, I); 534 case TargetOpcode::G_BITREVERSE: 535 return selectBitreverse(ResVReg, ResType, I); 536 537 case TargetOpcode::G_BUILD_VECTOR: 538 return selectBuildVector(ResVReg, ResType, I); 539 case TargetOpcode::G_SPLAT_VECTOR: 540 return selectSplatVector(ResVReg, ResType, I); 541 542 case TargetOpcode::G_SHUFFLE_VECTOR: { 543 MachineBasicBlock &BB = *I.getParent(); 544 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 545 .addDef(ResVReg) 546 .addUse(GR.getSPIRVTypeID(ResType)) 547 .addUse(I.getOperand(1).getReg()) 548 .addUse(I.getOperand(2).getReg()); 549 for (auto V : I.getOperand(3).getShuffleMask()) 550 MIB.addImm(V); 551 return MIB.constrainAllUses(TII, TRI, RBI); 552 } 553 case TargetOpcode::G_MEMMOVE: 554 case TargetOpcode::G_MEMCPY: 555 case TargetOpcode::G_MEMSET: 556 return selectMemOperation(ResVReg, I); 557 558 case TargetOpcode::G_ICMP: 559 return selectICmp(ResVReg, ResType, I); 560 case TargetOpcode::G_FCMP: 561 return selectFCmp(ResVReg, ResType, I); 562 563 case TargetOpcode::G_FRAME_INDEX: 564 return selectFrameIndex(ResVReg, ResType, I); 565 566 case TargetOpcode::G_LOAD: 567 return selectLoad(ResVReg, ResType, I); 568 case TargetOpcode::G_STORE: 569 return selectStore(I); 570 571 case TargetOpcode::G_BR: 572 return selectBranch(I); 573 case TargetOpcode::G_BRCOND: 574 return selectBranchCond(I); 575 576 case TargetOpcode::G_PHI: 577 return selectPhi(ResVReg, ResType, I); 578 579 case TargetOpcode::G_FPTOSI: 580 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 581 case TargetOpcode::G_FPTOUI: 582 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 583 584 case TargetOpcode::G_SITOFP: 585 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 586 case TargetOpcode::G_UITOFP: 587 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 588 589 case TargetOpcode::G_CTPOP: 590 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 591 case TargetOpcode::G_SMIN: 592 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 593 case TargetOpcode::G_UMIN: 594 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 595 596 case TargetOpcode::G_SMAX: 597 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 598 case TargetOpcode::G_UMAX: 599 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 600 601 case TargetOpcode::G_SCMP: 602 return selectSUCmp(ResVReg, ResType, I, true); 603 case TargetOpcode::G_UCMP: 604 return selectSUCmp(ResVReg, ResType, I, false); 605 606 case TargetOpcode::G_STRICT_FMA: 607 case TargetOpcode::G_FMA: 608 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 609 610 case TargetOpcode::G_STRICT_FLDEXP: 611 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 612 613 case TargetOpcode::G_FPOW: 614 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 615 case TargetOpcode::G_FPOWI: 616 return selectExtInst(ResVReg, ResType, I, CL::pown); 617 618 case TargetOpcode::G_FEXP: 619 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 620 case TargetOpcode::G_FEXP2: 621 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 622 623 case TargetOpcode::G_FLOG: 624 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 625 case TargetOpcode::G_FLOG2: 626 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 627 case TargetOpcode::G_FLOG10: 628 return selectLog10(ResVReg, ResType, I); 629 630 case TargetOpcode::G_FABS: 631 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 632 case TargetOpcode::G_ABS: 633 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 634 635 case TargetOpcode::G_FMINNUM: 636 case TargetOpcode::G_FMINIMUM: 637 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 638 case TargetOpcode::G_FMAXNUM: 639 case TargetOpcode::G_FMAXIMUM: 640 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 641 642 case TargetOpcode::G_FCOPYSIGN: 643 return selectExtInst(ResVReg, ResType, I, CL::copysign); 644 645 case TargetOpcode::G_FCEIL: 646 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 647 case TargetOpcode::G_FFLOOR: 648 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 649 650 case TargetOpcode::G_FCOS: 651 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 652 case TargetOpcode::G_FSIN: 653 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 654 case TargetOpcode::G_FTAN: 655 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 656 case TargetOpcode::G_FACOS: 657 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 658 case TargetOpcode::G_FASIN: 659 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 660 case TargetOpcode::G_FATAN: 661 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 662 case TargetOpcode::G_FATAN2: 663 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 664 case TargetOpcode::G_FCOSH: 665 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 666 case TargetOpcode::G_FSINH: 667 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 668 case TargetOpcode::G_FTANH: 669 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 670 671 case TargetOpcode::G_STRICT_FSQRT: 672 case TargetOpcode::G_FSQRT: 673 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 674 675 case TargetOpcode::G_CTTZ: 676 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 677 return selectExtInst(ResVReg, ResType, I, CL::ctz); 678 case TargetOpcode::G_CTLZ: 679 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 680 return selectExtInst(ResVReg, ResType, I, CL::clz); 681 682 case TargetOpcode::G_INTRINSIC_ROUND: 683 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 684 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 685 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 686 case TargetOpcode::G_INTRINSIC_TRUNC: 687 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 688 case TargetOpcode::G_FRINT: 689 case TargetOpcode::G_FNEARBYINT: 690 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 691 692 case TargetOpcode::G_SMULH: 693 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 694 case TargetOpcode::G_UMULH: 695 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 696 697 case TargetOpcode::G_SADDSAT: 698 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 699 case TargetOpcode::G_UADDSAT: 700 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 701 case TargetOpcode::G_SSUBSAT: 702 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 703 case TargetOpcode::G_USUBSAT: 704 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 705 706 case TargetOpcode::G_UADDO: 707 return selectOverflowArith(ResVReg, ResType, I, 708 ResType->getOpcode() == SPIRV::OpTypeVector 709 ? SPIRV::OpIAddCarryV 710 : SPIRV::OpIAddCarryS); 711 case TargetOpcode::G_USUBO: 712 return selectOverflowArith(ResVReg, ResType, I, 713 ResType->getOpcode() == SPIRV::OpTypeVector 714 ? SPIRV::OpISubBorrowV 715 : SPIRV::OpISubBorrowS); 716 case TargetOpcode::G_UMULO: 717 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 718 case TargetOpcode::G_SMULO: 719 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 720 721 case TargetOpcode::G_SEXT: 722 return selectExt(ResVReg, ResType, I, true); 723 case TargetOpcode::G_ANYEXT: 724 case TargetOpcode::G_ZEXT: 725 return selectExt(ResVReg, ResType, I, false); 726 case TargetOpcode::G_TRUNC: 727 return selectTrunc(ResVReg, ResType, I); 728 case TargetOpcode::G_FPTRUNC: 729 case TargetOpcode::G_FPEXT: 730 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 731 732 case TargetOpcode::G_PTRTOINT: 733 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 734 case TargetOpcode::G_INTTOPTR: 735 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 736 case TargetOpcode::G_BITCAST: 737 return selectBitcast(ResVReg, ResType, I); 738 case TargetOpcode::G_ADDRSPACE_CAST: 739 return selectAddrSpaceCast(ResVReg, ResType, I); 740 case TargetOpcode::G_PTR_ADD: { 741 // Currently, we get G_PTR_ADD only applied to global variables. 742 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 743 Register GV = I.getOperand(1).getReg(); 744 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 745 (void)II; 746 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 747 (*II).getOpcode() == TargetOpcode::COPY || 748 (*II).getOpcode() == SPIRV::OpVariable) && 749 isImm(I.getOperand(2), MRI)); 750 // It may be the initialization of a global variable. 751 bool IsGVInit = false; 752 for (MachineRegisterInfo::use_instr_iterator 753 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 754 UseEnd = MRI->use_instr_end(); 755 UseIt != UseEnd; UseIt = std::next(UseIt)) { 756 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 757 (*UseIt).getOpcode() == SPIRV::OpVariable) { 758 IsGVInit = true; 759 break; 760 } 761 } 762 MachineBasicBlock &BB = *I.getParent(); 763 if (!IsGVInit) { 764 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 765 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 766 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 767 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 768 // Build a new virtual register that is associated with the required 769 // data type. 770 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 771 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 772 // Having a correctly typed base we are ready to build the actually 773 // required GEP. It may not be a constant though, because all Operands 774 // of OpSpecConstantOp is to originate from other const instructions, 775 // and only the AccessChain named opcodes accept a global OpVariable 776 // instruction. We can't use an AccessChain opcode because of the type 777 // mismatch between result and base types. 778 if (!GR.isBitcastCompatible(ResType, GVType)) 779 report_fatal_error( 780 "incompatible result and operand types in a bitcast"); 781 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 782 MachineInstrBuilder MIB = 783 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 784 .addDef(NewVReg) 785 .addUse(ResTypeReg) 786 .addUse(GV); 787 return MIB.constrainAllUses(TII, TRI, RBI) && 788 BuildMI(BB, I, I.getDebugLoc(), 789 TII.get(STI.isVulkanEnv() 790 ? SPIRV::OpInBoundsAccessChain 791 : SPIRV::OpInBoundsPtrAccessChain)) 792 .addDef(ResVReg) 793 .addUse(ResTypeReg) 794 .addUse(NewVReg) 795 .addUse(I.getOperand(2).getReg()) 796 .constrainAllUses(TII, TRI, RBI); 797 } else { 798 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 799 .addDef(ResVReg) 800 .addUse(GR.getSPIRVTypeID(ResType)) 801 .addImm( 802 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 803 .addUse(GV) 804 .addUse(I.getOperand(2).getReg()) 805 .constrainAllUses(TII, TRI, RBI); 806 } 807 } 808 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 809 // initialize a global variable with a constant expression (e.g., the test 810 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 811 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 812 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 813 .addDef(ResVReg) 814 .addUse(GR.getSPIRVTypeID(ResType)) 815 .addImm(static_cast<uint32_t>( 816 SPIRV::Opcode::InBoundsPtrAccessChain)) 817 .addUse(GV) 818 .addUse(Idx) 819 .addUse(I.getOperand(2).getReg()); 820 return MIB.constrainAllUses(TII, TRI, RBI); 821 } 822 823 case TargetOpcode::G_ATOMICRMW_OR: 824 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 825 case TargetOpcode::G_ATOMICRMW_ADD: 826 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 827 case TargetOpcode::G_ATOMICRMW_AND: 828 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 829 case TargetOpcode::G_ATOMICRMW_MAX: 830 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 831 case TargetOpcode::G_ATOMICRMW_MIN: 832 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 833 case TargetOpcode::G_ATOMICRMW_SUB: 834 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 835 case TargetOpcode::G_ATOMICRMW_XOR: 836 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 837 case TargetOpcode::G_ATOMICRMW_UMAX: 838 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 839 case TargetOpcode::G_ATOMICRMW_UMIN: 840 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 841 case TargetOpcode::G_ATOMICRMW_XCHG: 842 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 843 case TargetOpcode::G_ATOMIC_CMPXCHG: 844 return selectAtomicCmpXchg(ResVReg, ResType, I); 845 846 case TargetOpcode::G_ATOMICRMW_FADD: 847 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 848 case TargetOpcode::G_ATOMICRMW_FSUB: 849 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 850 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 851 SPIRV::OpFNegate); 852 case TargetOpcode::G_ATOMICRMW_FMIN: 853 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 854 case TargetOpcode::G_ATOMICRMW_FMAX: 855 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 856 857 case TargetOpcode::G_FENCE: 858 return selectFence(I); 859 860 case TargetOpcode::G_STACKSAVE: 861 return selectStackSave(ResVReg, ResType, I); 862 case TargetOpcode::G_STACKRESTORE: 863 return selectStackRestore(I); 864 865 case TargetOpcode::G_UNMERGE_VALUES: 866 return selectUnmergeValues(I); 867 868 // Discard gen opcodes for intrinsics which we do not expect to actually 869 // represent code after lowering or intrinsics which are not implemented but 870 // should not crash when found in a customer's LLVM IR input. 871 case TargetOpcode::G_TRAP: 872 case TargetOpcode::G_DEBUGTRAP: 873 case TargetOpcode::G_UBSANTRAP: 874 case TargetOpcode::DBG_LABEL: 875 return true; 876 877 default: 878 return false; 879 } 880 } 881 882 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 883 const SPIRVType *ResType, 884 MachineInstr &I, 885 GL::GLSLExtInst GLInst) const { 886 return selectExtInst(ResVReg, ResType, I, 887 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 888 } 889 890 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 891 const SPIRVType *ResType, 892 MachineInstr &I, 893 CL::OpenCLExtInst CLInst) const { 894 return selectExtInst(ResVReg, ResType, I, 895 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 896 } 897 898 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 899 const SPIRVType *ResType, 900 MachineInstr &I, 901 CL::OpenCLExtInst CLInst, 902 GL::GLSLExtInst GLInst) const { 903 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 904 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 905 return selectExtInst(ResVReg, ResType, I, ExtInsts); 906 } 907 908 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 909 const SPIRVType *ResType, 910 MachineInstr &I, 911 const ExtInstList &Insts) const { 912 913 for (const auto &Ex : Insts) { 914 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 915 uint32_t Opcode = Ex.second; 916 if (STI.canUseExtInstSet(Set)) { 917 MachineBasicBlock &BB = *I.getParent(); 918 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 919 .addDef(ResVReg) 920 .addUse(GR.getSPIRVTypeID(ResType)) 921 .addImm(static_cast<uint32_t>(Set)) 922 .addImm(Opcode); 923 const unsigned NumOps = I.getNumOperands(); 924 unsigned Index = 1; 925 if (Index < NumOps && 926 I.getOperand(Index).getType() == 927 MachineOperand::MachineOperandType::MO_IntrinsicID) 928 Index = 2; 929 for (; Index < NumOps; ++Index) 930 MIB.add(I.getOperand(Index)); 931 return MIB.constrainAllUses(TII, TRI, RBI); 932 } 933 } 934 return false; 935 } 936 937 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 938 const SPIRVType *ResType, 939 MachineInstr &I, 940 std::vector<Register> Srcs, 941 unsigned Opcode) const { 942 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 943 .addDef(ResVReg) 944 .addUse(GR.getSPIRVTypeID(ResType)); 945 for (Register SReg : Srcs) { 946 MIB.addUse(SReg); 947 } 948 return MIB.constrainAllUses(TII, TRI, RBI); 949 } 950 951 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 952 const SPIRVType *ResType, 953 MachineInstr &I, 954 unsigned Opcode) const { 955 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 956 Register SrcReg = I.getOperand(1).getReg(); 957 bool IsGV = false; 958 for (MachineRegisterInfo::def_instr_iterator DefIt = 959 MRI->def_instr_begin(SrcReg); 960 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 961 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 962 IsGV = true; 963 break; 964 } 965 } 966 if (IsGV) { 967 uint32_t SpecOpcode = 0; 968 switch (Opcode) { 969 case SPIRV::OpConvertPtrToU: 970 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 971 break; 972 case SPIRV::OpConvertUToPtr: 973 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 974 break; 975 } 976 if (SpecOpcode) 977 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 978 TII.get(SPIRV::OpSpecConstantOp)) 979 .addDef(ResVReg) 980 .addUse(GR.getSPIRVTypeID(ResType)) 981 .addImm(SpecOpcode) 982 .addUse(SrcReg) 983 .constrainAllUses(TII, TRI, RBI); 984 } 985 } 986 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 987 Opcode); 988 } 989 990 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 991 const SPIRVType *ResType, 992 MachineInstr &I) const { 993 Register OpReg = I.getOperand(1).getReg(); 994 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 995 if (!GR.isBitcastCompatible(ResType, OpType)) 996 report_fatal_error("incompatible result and operand types in a bitcast"); 997 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 998 } 999 1000 static void addMemoryOperands(MachineMemOperand *MemOp, 1001 MachineInstrBuilder &MIB) { 1002 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1003 if (MemOp->isVolatile()) 1004 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1005 if (MemOp->isNonTemporal()) 1006 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1007 if (MemOp->getAlign().value()) 1008 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 1009 1010 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 1011 MIB.addImm(SpvMemOp); 1012 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 1013 MIB.addImm(MemOp->getAlign().value()); 1014 } 1015 } 1016 1017 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1018 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1019 if (Flags & MachineMemOperand::Flags::MOVolatile) 1020 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1021 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1022 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1023 1024 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1025 MIB.addImm(SpvMemOp); 1026 } 1027 1028 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1029 const SPIRVType *ResType, 1030 MachineInstr &I) const { 1031 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1032 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1033 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1034 .addDef(ResVReg) 1035 .addUse(GR.getSPIRVTypeID(ResType)) 1036 .addUse(Ptr); 1037 if (!I.getNumMemOperands()) { 1038 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1039 I.getOpcode() == 1040 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1041 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1042 } else { 1043 addMemoryOperands(*I.memoperands_begin(), MIB); 1044 } 1045 return MIB.constrainAllUses(TII, TRI, RBI); 1046 } 1047 1048 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1049 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1050 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1051 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1052 MachineBasicBlock &BB = *I.getParent(); 1053 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1054 .addUse(Ptr) 1055 .addUse(StoreVal); 1056 if (!I.getNumMemOperands()) { 1057 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1058 I.getOpcode() == 1059 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1060 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1061 } else { 1062 addMemoryOperands(*I.memoperands_begin(), MIB); 1063 } 1064 return MIB.constrainAllUses(TII, TRI, RBI); 1065 } 1066 1067 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1068 const SPIRVType *ResType, 1069 MachineInstr &I) const { 1070 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1071 report_fatal_error( 1072 "llvm.stacksave intrinsic: this instruction requires the following " 1073 "SPIR-V extension: SPV_INTEL_variable_length_array", 1074 false); 1075 MachineBasicBlock &BB = *I.getParent(); 1076 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1077 .addDef(ResVReg) 1078 .addUse(GR.getSPIRVTypeID(ResType)) 1079 .constrainAllUses(TII, TRI, RBI); 1080 } 1081 1082 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1083 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1084 report_fatal_error( 1085 "llvm.stackrestore intrinsic: this instruction requires the following " 1086 "SPIR-V extension: SPV_INTEL_variable_length_array", 1087 false); 1088 if (!I.getOperand(0).isReg()) 1089 return false; 1090 MachineBasicBlock &BB = *I.getParent(); 1091 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1092 .addUse(I.getOperand(0).getReg()) 1093 .constrainAllUses(TII, TRI, RBI); 1094 } 1095 1096 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1097 MachineInstr &I) const { 1098 MachineBasicBlock &BB = *I.getParent(); 1099 Register SrcReg = I.getOperand(1).getReg(); 1100 bool Result = true; 1101 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1102 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1103 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1104 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1105 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1106 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1107 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1108 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1109 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1110 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1111 Function &CurFunction = GR.CurMF->getFunction(); 1112 Type *LLVMArrTy = 1113 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1114 // Module takes ownership of the global var. 1115 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1116 true, GlobalValue::InternalLinkage, 1117 Constant::getNullValue(LLVMArrTy)); 1118 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1119 GR.add(GV, GR.CurMF, VarReg); 1120 GR.addGlobalObject(GV, GR.CurMF, VarReg); 1121 1122 Result &= 1123 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1124 .addDef(VarReg) 1125 .addUse(GR.getSPIRVTypeID(VarTy)) 1126 .addImm(SPIRV::StorageClass::UniformConstant) 1127 .addUse(Const) 1128 .constrainAllUses(TII, TRI, RBI); 1129 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1130 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1131 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1132 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1133 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1134 } 1135 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1136 .addUse(I.getOperand(0).getReg()) 1137 .addUse(SrcReg) 1138 .addUse(I.getOperand(2).getReg()); 1139 if (I.getNumMemOperands()) 1140 addMemoryOperands(*I.memoperands_begin(), MIB); 1141 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1142 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1143 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1144 return Result; 1145 } 1146 1147 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1148 const SPIRVType *ResType, 1149 MachineInstr &I, 1150 unsigned NewOpcode, 1151 unsigned NegateOpcode) const { 1152 bool Result = true; 1153 assert(I.hasOneMemOperand()); 1154 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1155 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1156 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1157 auto ScopeConstant = buildI32Constant(Scope, I); 1158 Register ScopeReg = ScopeConstant.first; 1159 Result &= ScopeConstant.second; 1160 1161 Register Ptr = I.getOperand(1).getReg(); 1162 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1163 // auto ScSem = 1164 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1165 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1166 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1167 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1168 Register MemSemReg = MemSemConstant.first; 1169 Result &= MemSemConstant.second; 1170 1171 Register ValueReg = I.getOperand(2).getReg(); 1172 if (NegateOpcode != 0) { 1173 // Translation with negative value operand is requested 1174 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1175 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1176 ValueReg = TmpReg; 1177 } 1178 1179 return Result && 1180 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1181 .addDef(ResVReg) 1182 .addUse(GR.getSPIRVTypeID(ResType)) 1183 .addUse(Ptr) 1184 .addUse(ScopeReg) 1185 .addUse(MemSemReg) 1186 .addUse(ValueReg) 1187 .constrainAllUses(TII, TRI, RBI); 1188 } 1189 1190 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1191 unsigned ArgI = I.getNumOperands() - 1; 1192 Register SrcReg = 1193 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1194 SPIRVType *DefType = 1195 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1196 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1197 report_fatal_error( 1198 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1199 1200 SPIRVType *ScalarType = 1201 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1202 MachineBasicBlock &BB = *I.getParent(); 1203 bool Res = false; 1204 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1205 Register ResVReg = I.getOperand(i).getReg(); 1206 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1207 if (!ResType) { 1208 // There was no "assign type" actions, let's fix this now 1209 ResType = ScalarType; 1210 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1211 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1212 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1213 } 1214 auto MIB = 1215 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1216 .addDef(ResVReg) 1217 .addUse(GR.getSPIRVTypeID(ResType)) 1218 .addUse(SrcReg) 1219 .addImm(static_cast<int64_t>(i)); 1220 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1221 } 1222 return Res; 1223 } 1224 1225 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1226 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1227 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1228 auto MemSemConstant = buildI32Constant(MemSem, I); 1229 Register MemSemReg = MemSemConstant.first; 1230 bool Result = MemSemConstant.second; 1231 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1232 uint32_t Scope = static_cast<uint32_t>( 1233 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1234 auto ScopeConstant = buildI32Constant(Scope, I); 1235 Register ScopeReg = ScopeConstant.first; 1236 Result &= ScopeConstant.second; 1237 MachineBasicBlock &BB = *I.getParent(); 1238 return Result && 1239 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1240 .addUse(ScopeReg) 1241 .addUse(MemSemReg) 1242 .constrainAllUses(TII, TRI, RBI); 1243 } 1244 1245 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1246 const SPIRVType *ResType, 1247 MachineInstr &I, 1248 unsigned Opcode) const { 1249 Type *ResTy = nullptr; 1250 StringRef ResName; 1251 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1252 report_fatal_error( 1253 "Not enough info to select the arithmetic with overflow instruction"); 1254 if (!ResTy || !ResTy->isStructTy()) 1255 report_fatal_error("Expect struct type result for the arithmetic " 1256 "with overflow instruction"); 1257 // "Result Type must be from OpTypeStruct. The struct must have two members, 1258 // and the two members must be the same type." 1259 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1260 ResTy = StructType::get(ResElemTy, ResElemTy); 1261 // Build SPIR-V types and constant(s) if needed. 1262 MachineIRBuilder MIRBuilder(I); 1263 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1264 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1265 assert(I.getNumDefs() > 1 && "Not enought operands"); 1266 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1267 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1268 if (N > 1) 1269 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1270 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1271 Register ZeroReg = buildZerosVal(ResType, I); 1272 // A new virtual register to store the result struct. 1273 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1274 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1275 // Build the result name if needed. 1276 if (ResName.size() > 0) 1277 buildOpName(StructVReg, ResName, MIRBuilder); 1278 // Build the arithmetic with overflow instruction. 1279 MachineBasicBlock &BB = *I.getParent(); 1280 auto MIB = 1281 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1282 .addDef(StructVReg) 1283 .addUse(GR.getSPIRVTypeID(StructType)); 1284 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1285 MIB.addUse(I.getOperand(i).getReg()); 1286 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1287 // Build instructions to extract fields of the instruction's result. 1288 // A new virtual register to store the higher part of the result struct. 1289 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1290 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1291 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1292 auto MIB = 1293 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1294 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1295 .addUse(GR.getSPIRVTypeID(ResType)) 1296 .addUse(StructVReg) 1297 .addImm(i); 1298 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1299 } 1300 // Build boolean value from the higher part. 1301 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1302 .addDef(I.getOperand(1).getReg()) 1303 .addUse(BoolTypeReg) 1304 .addUse(HigherVReg) 1305 .addUse(ZeroReg) 1306 .constrainAllUses(TII, TRI, RBI); 1307 } 1308 1309 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1310 const SPIRVType *ResType, 1311 MachineInstr &I) const { 1312 bool Result = true; 1313 Register ScopeReg; 1314 Register MemSemEqReg; 1315 Register MemSemNeqReg; 1316 Register Ptr = I.getOperand(2).getReg(); 1317 if (!isa<GIntrinsic>(I)) { 1318 assert(I.hasOneMemOperand()); 1319 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1320 unsigned Scope = static_cast<uint32_t>(getMemScope( 1321 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1322 auto ScopeConstant = buildI32Constant(Scope, I); 1323 ScopeReg = ScopeConstant.first; 1324 Result &= ScopeConstant.second; 1325 1326 unsigned ScSem = static_cast<uint32_t>( 1327 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1328 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1329 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1330 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1331 MemSemEqReg = MemSemEqConstant.first; 1332 Result &= MemSemEqConstant.second; 1333 AtomicOrdering FO = MemOp->getFailureOrdering(); 1334 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1335 if (MemSemEq == MemSemNeq) 1336 MemSemNeqReg = MemSemEqReg; 1337 else { 1338 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1339 MemSemNeqReg = MemSemNeqConstant.first; 1340 Result &= MemSemNeqConstant.second; 1341 } 1342 } else { 1343 ScopeReg = I.getOperand(5).getReg(); 1344 MemSemEqReg = I.getOperand(6).getReg(); 1345 MemSemNeqReg = I.getOperand(7).getReg(); 1346 } 1347 1348 Register Cmp = I.getOperand(3).getReg(); 1349 Register Val = I.getOperand(4).getReg(); 1350 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1351 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1352 const DebugLoc &DL = I.getDebugLoc(); 1353 Result &= 1354 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1355 .addDef(ACmpRes) 1356 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1357 .addUse(Ptr) 1358 .addUse(ScopeReg) 1359 .addUse(MemSemEqReg) 1360 .addUse(MemSemNeqReg) 1361 .addUse(Val) 1362 .addUse(Cmp) 1363 .constrainAllUses(TII, TRI, RBI); 1364 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1365 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1366 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1367 .addDef(CmpSuccReg) 1368 .addUse(GR.getSPIRVTypeID(BoolTy)) 1369 .addUse(ACmpRes) 1370 .addUse(Cmp) 1371 .constrainAllUses(TII, TRI, RBI); 1372 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1373 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1374 .addDef(TmpReg) 1375 .addUse(GR.getSPIRVTypeID(ResType)) 1376 .addUse(ACmpRes) 1377 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1378 .addImm(0) 1379 .constrainAllUses(TII, TRI, RBI); 1380 return Result && 1381 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1382 .addDef(ResVReg) 1383 .addUse(GR.getSPIRVTypeID(ResType)) 1384 .addUse(CmpSuccReg) 1385 .addUse(TmpReg) 1386 .addImm(1) 1387 .constrainAllUses(TII, TRI, RBI); 1388 } 1389 1390 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1391 switch (SC) { 1392 case SPIRV::StorageClass::Workgroup: 1393 case SPIRV::StorageClass::CrossWorkgroup: 1394 case SPIRV::StorageClass::Function: 1395 return true; 1396 default: 1397 return false; 1398 } 1399 } 1400 1401 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1402 switch (SC) { 1403 case SPIRV::StorageClass::DeviceOnlyINTEL: 1404 case SPIRV::StorageClass::HostOnlyINTEL: 1405 return true; 1406 default: 1407 return false; 1408 } 1409 } 1410 1411 // Returns true ResVReg is referred only from global vars and OpName's. 1412 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1413 bool IsGRef = false; 1414 bool IsAllowedRefs = 1415 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1416 [&IsGRef](auto const &It) { 1417 unsigned Opcode = It.getOpcode(); 1418 if (Opcode == SPIRV::OpConstantComposite || 1419 Opcode == SPIRV::OpVariable || 1420 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1421 return IsGRef = true; 1422 return Opcode == SPIRV::OpName; 1423 }); 1424 return IsAllowedRefs && IsGRef; 1425 } 1426 1427 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1428 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1429 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1430 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1431 } 1432 1433 MachineInstrBuilder 1434 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1435 Register Src, Register DestType, 1436 uint32_t Opcode) const { 1437 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1438 TII.get(SPIRV::OpSpecConstantOp)) 1439 .addDef(Dest) 1440 .addUse(DestType) 1441 .addImm(Opcode) 1442 .addUse(Src); 1443 } 1444 1445 MachineInstrBuilder 1446 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1447 SPIRVType *SrcPtrTy) const { 1448 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1449 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1450 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1451 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1452 SPIRV::StorageClass::Generic), 1453 GR.getPointerSize())); 1454 MachineFunction *MF = I.getParent()->getParent(); 1455 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1456 MachineInstrBuilder MIB = buildSpecConstantOp( 1457 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1458 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1459 GR.add(MIB.getInstr(), MF, Tmp); 1460 return MIB; 1461 } 1462 1463 // In SPIR-V address space casting can only happen to and from the Generic 1464 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1465 // pointers to and from Generic pointers. As such, we can convert e.g. from 1466 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1467 // other combinations can only be done by a bitcast, and are probably not safe. 1468 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1469 const SPIRVType *ResType, 1470 MachineInstr &I) const { 1471 MachineBasicBlock &BB = *I.getParent(); 1472 const DebugLoc &DL = I.getDebugLoc(); 1473 1474 Register SrcPtr = I.getOperand(1).getReg(); 1475 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1476 1477 // don't generate a cast for a null that may be represented by OpTypeInt 1478 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1479 ResType->getOpcode() != SPIRV::OpTypePointer) 1480 return BuildCOPY(ResVReg, SrcPtr, I); 1481 1482 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1483 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1484 1485 if (isASCastInGVar(MRI, ResVReg)) { 1486 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1487 // are expressed by OpSpecConstantOp with an Opcode. 1488 // TODO: maybe insert a check whether the Kernel capability was declared and 1489 // so PtrCastToGeneric/GenericCastToPtr are available. 1490 unsigned SpecOpcode = 1491 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1492 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1493 : (SrcSC == SPIRV::StorageClass::Generic && 1494 isGenericCastablePtr(DstSC) 1495 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1496 : 0); 1497 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1498 // correct value of ResType and use general i8* instead. Maybe this should 1499 // be addressed in the emit-intrinsic step to infer a correct 1500 // OpConstantComposite type. 1501 if (SpecOpcode) { 1502 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1503 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1504 .constrainAllUses(TII, TRI, RBI); 1505 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1506 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1507 return MIB.constrainAllUses(TII, TRI, RBI) && 1508 buildSpecConstantOp( 1509 I, ResVReg, MIB->getOperand(0).getReg(), 1510 getUcharPtrTypeReg(I, DstSC), 1511 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1512 .constrainAllUses(TII, TRI, RBI); 1513 } 1514 } 1515 1516 // don't generate a cast between identical storage classes 1517 if (SrcSC == DstSC) 1518 return BuildCOPY(ResVReg, SrcPtr, I); 1519 1520 if ((SrcSC == SPIRV::StorageClass::Function && 1521 DstSC == SPIRV::StorageClass::Private) || 1522 (DstSC == SPIRV::StorageClass::Function && 1523 SrcSC == SPIRV::StorageClass::Private)) 1524 return BuildCOPY(ResVReg, SrcPtr, I); 1525 1526 // Casting from an eligible pointer to Generic. 1527 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1528 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1529 // Casting from Generic to an eligible pointer. 1530 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1531 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1532 // Casting between 2 eligible pointers using Generic as an intermediary. 1533 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1534 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1535 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1536 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1537 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1538 .addDef(Tmp) 1539 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1540 .addUse(SrcPtr) 1541 .constrainAllUses(TII, TRI, RBI); 1542 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1543 .addDef(ResVReg) 1544 .addUse(GR.getSPIRVTypeID(ResType)) 1545 .addUse(Tmp) 1546 .constrainAllUses(TII, TRI, RBI); 1547 } 1548 1549 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1550 // be applied 1551 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1552 return selectUnOp(ResVReg, ResType, I, 1553 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1554 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1555 return selectUnOp(ResVReg, ResType, I, 1556 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1557 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1558 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1559 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1560 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1561 1562 // Bitcast for pointers requires that the address spaces must match 1563 return false; 1564 } 1565 1566 static unsigned getFCmpOpcode(unsigned PredNum) { 1567 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1568 switch (Pred) { 1569 case CmpInst::FCMP_OEQ: 1570 return SPIRV::OpFOrdEqual; 1571 case CmpInst::FCMP_OGE: 1572 return SPIRV::OpFOrdGreaterThanEqual; 1573 case CmpInst::FCMP_OGT: 1574 return SPIRV::OpFOrdGreaterThan; 1575 case CmpInst::FCMP_OLE: 1576 return SPIRV::OpFOrdLessThanEqual; 1577 case CmpInst::FCMP_OLT: 1578 return SPIRV::OpFOrdLessThan; 1579 case CmpInst::FCMP_ONE: 1580 return SPIRV::OpFOrdNotEqual; 1581 case CmpInst::FCMP_ORD: 1582 return SPIRV::OpOrdered; 1583 case CmpInst::FCMP_UEQ: 1584 return SPIRV::OpFUnordEqual; 1585 case CmpInst::FCMP_UGE: 1586 return SPIRV::OpFUnordGreaterThanEqual; 1587 case CmpInst::FCMP_UGT: 1588 return SPIRV::OpFUnordGreaterThan; 1589 case CmpInst::FCMP_ULE: 1590 return SPIRV::OpFUnordLessThanEqual; 1591 case CmpInst::FCMP_ULT: 1592 return SPIRV::OpFUnordLessThan; 1593 case CmpInst::FCMP_UNE: 1594 return SPIRV::OpFUnordNotEqual; 1595 case CmpInst::FCMP_UNO: 1596 return SPIRV::OpUnordered; 1597 default: 1598 llvm_unreachable("Unknown predicate type for FCmp"); 1599 } 1600 } 1601 1602 static unsigned getICmpOpcode(unsigned PredNum) { 1603 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1604 switch (Pred) { 1605 case CmpInst::ICMP_EQ: 1606 return SPIRV::OpIEqual; 1607 case CmpInst::ICMP_NE: 1608 return SPIRV::OpINotEqual; 1609 case CmpInst::ICMP_SGE: 1610 return SPIRV::OpSGreaterThanEqual; 1611 case CmpInst::ICMP_SGT: 1612 return SPIRV::OpSGreaterThan; 1613 case CmpInst::ICMP_SLE: 1614 return SPIRV::OpSLessThanEqual; 1615 case CmpInst::ICMP_SLT: 1616 return SPIRV::OpSLessThan; 1617 case CmpInst::ICMP_UGE: 1618 return SPIRV::OpUGreaterThanEqual; 1619 case CmpInst::ICMP_UGT: 1620 return SPIRV::OpUGreaterThan; 1621 case CmpInst::ICMP_ULE: 1622 return SPIRV::OpULessThanEqual; 1623 case CmpInst::ICMP_ULT: 1624 return SPIRV::OpULessThan; 1625 default: 1626 llvm_unreachable("Unknown predicate type for ICmp"); 1627 } 1628 } 1629 1630 static unsigned getPtrCmpOpcode(unsigned Pred) { 1631 switch (static_cast<CmpInst::Predicate>(Pred)) { 1632 case CmpInst::ICMP_EQ: 1633 return SPIRV::OpPtrEqual; 1634 case CmpInst::ICMP_NE: 1635 return SPIRV::OpPtrNotEqual; 1636 default: 1637 llvm_unreachable("Unknown predicate type for pointer comparison"); 1638 } 1639 } 1640 1641 // Return the logical operation, or abort if none exists. 1642 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1643 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1644 switch (Pred) { 1645 case CmpInst::ICMP_EQ: 1646 return SPIRV::OpLogicalEqual; 1647 case CmpInst::ICMP_NE: 1648 return SPIRV::OpLogicalNotEqual; 1649 default: 1650 llvm_unreachable("Unknown predicate type for Bool comparison"); 1651 } 1652 } 1653 1654 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1655 if (!LLVMFloatTy) 1656 return APFloat::getZero(APFloat::IEEEsingle()); 1657 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1658 case Type::HalfTyID: 1659 return APFloat::getZero(APFloat::IEEEhalf()); 1660 default: 1661 case Type::FloatTyID: 1662 return APFloat::getZero(APFloat::IEEEsingle()); 1663 case Type::DoubleTyID: 1664 return APFloat::getZero(APFloat::IEEEdouble()); 1665 } 1666 } 1667 1668 static APFloat getOneFP(const Type *LLVMFloatTy) { 1669 if (!LLVMFloatTy) 1670 return APFloat::getOne(APFloat::IEEEsingle()); 1671 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1672 case Type::HalfTyID: 1673 return APFloat::getOne(APFloat::IEEEhalf()); 1674 default: 1675 case Type::FloatTyID: 1676 return APFloat::getOne(APFloat::IEEEsingle()); 1677 case Type::DoubleTyID: 1678 return APFloat::getOne(APFloat::IEEEdouble()); 1679 } 1680 } 1681 1682 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1683 const SPIRVType *ResType, 1684 MachineInstr &I, 1685 unsigned OpAnyOrAll) const { 1686 assert(I.getNumOperands() == 3); 1687 assert(I.getOperand(2).isReg()); 1688 MachineBasicBlock &BB = *I.getParent(); 1689 Register InputRegister = I.getOperand(2).getReg(); 1690 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1691 1692 if (!InputType) 1693 report_fatal_error("Input Type could not be determined."); 1694 1695 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1696 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1697 if (IsBoolTy && !IsVectorTy) { 1698 assert(ResVReg == I.getOperand(0).getReg()); 1699 return BuildCOPY(ResVReg, InputRegister, I); 1700 } 1701 1702 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1703 unsigned SpirvNotEqualId = 1704 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1705 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1706 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1707 Register NotEqualReg = ResVReg; 1708 1709 if (IsVectorTy) { 1710 NotEqualReg = IsBoolTy ? InputRegister 1711 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1712 const unsigned NumElts = InputType->getOperand(2).getImm(); 1713 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1714 } 1715 1716 bool Result = true; 1717 if (!IsBoolTy) { 1718 Register ConstZeroReg = 1719 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1720 1721 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1722 .addDef(NotEqualReg) 1723 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1724 .addUse(InputRegister) 1725 .addUse(ConstZeroReg) 1726 .constrainAllUses(TII, TRI, RBI); 1727 } 1728 1729 if (!IsVectorTy) 1730 return Result; 1731 1732 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1733 .addDef(ResVReg) 1734 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1735 .addUse(NotEqualReg) 1736 .constrainAllUses(TII, TRI, RBI); 1737 } 1738 1739 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1740 const SPIRVType *ResType, 1741 MachineInstr &I) const { 1742 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1743 } 1744 1745 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1746 const SPIRVType *ResType, 1747 MachineInstr &I) const { 1748 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1749 } 1750 1751 // Select the OpDot instruction for the given float dot 1752 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1753 const SPIRVType *ResType, 1754 MachineInstr &I) const { 1755 assert(I.getNumOperands() == 4); 1756 assert(I.getOperand(2).isReg()); 1757 assert(I.getOperand(3).isReg()); 1758 1759 [[maybe_unused]] SPIRVType *VecType = 1760 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1761 1762 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1763 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1764 "dot product requires a vector of at least 2 components"); 1765 1766 [[maybe_unused]] SPIRVType *EltType = 1767 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1768 1769 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1770 1771 MachineBasicBlock &BB = *I.getParent(); 1772 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1773 .addDef(ResVReg) 1774 .addUse(GR.getSPIRVTypeID(ResType)) 1775 .addUse(I.getOperand(2).getReg()) 1776 .addUse(I.getOperand(3).getReg()) 1777 .constrainAllUses(TII, TRI, RBI); 1778 } 1779 1780 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1781 const SPIRVType *ResType, 1782 MachineInstr &I, 1783 bool Signed) const { 1784 assert(I.getNumOperands() == 4); 1785 assert(I.getOperand(2).isReg()); 1786 assert(I.getOperand(3).isReg()); 1787 MachineBasicBlock &BB = *I.getParent(); 1788 1789 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1790 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1791 .addDef(ResVReg) 1792 .addUse(GR.getSPIRVTypeID(ResType)) 1793 .addUse(I.getOperand(2).getReg()) 1794 .addUse(I.getOperand(3).getReg()) 1795 .constrainAllUses(TII, TRI, RBI); 1796 } 1797 1798 // Since pre-1.6 SPIRV has no integer dot implementation, 1799 // expand by piecewise multiplying and adding the results 1800 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1801 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1802 assert(I.getNumOperands() == 4); 1803 assert(I.getOperand(2).isReg()); 1804 assert(I.getOperand(3).isReg()); 1805 MachineBasicBlock &BB = *I.getParent(); 1806 1807 // Multiply the vectors, then sum the results 1808 Register Vec0 = I.getOperand(2).getReg(); 1809 Register Vec1 = I.getOperand(3).getReg(); 1810 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1811 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1812 1813 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1814 .addDef(TmpVec) 1815 .addUse(GR.getSPIRVTypeID(VecType)) 1816 .addUse(Vec0) 1817 .addUse(Vec1) 1818 .constrainAllUses(TII, TRI, RBI); 1819 1820 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1821 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1822 "dot product requires a vector of at least 2 components"); 1823 1824 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1825 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1826 .addDef(Res) 1827 .addUse(GR.getSPIRVTypeID(ResType)) 1828 .addUse(TmpVec) 1829 .addImm(0) 1830 .constrainAllUses(TII, TRI, RBI); 1831 1832 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1833 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1834 1835 Result &= 1836 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1837 .addDef(Elt) 1838 .addUse(GR.getSPIRVTypeID(ResType)) 1839 .addUse(TmpVec) 1840 .addImm(i) 1841 .constrainAllUses(TII, TRI, RBI); 1842 1843 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1844 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1845 : ResVReg; 1846 1847 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1848 .addDef(Sum) 1849 .addUse(GR.getSPIRVTypeID(ResType)) 1850 .addUse(Res) 1851 .addUse(Elt) 1852 .constrainAllUses(TII, TRI, RBI); 1853 Res = Sum; 1854 } 1855 1856 return Result; 1857 } 1858 1859 template <bool Signed> 1860 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1861 const SPIRVType *ResType, 1862 MachineInstr &I) const { 1863 assert(I.getNumOperands() == 5); 1864 assert(I.getOperand(2).isReg()); 1865 assert(I.getOperand(3).isReg()); 1866 assert(I.getOperand(4).isReg()); 1867 MachineBasicBlock &BB = *I.getParent(); 1868 1869 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1870 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1871 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1872 .addDef(Dot) 1873 .addUse(GR.getSPIRVTypeID(ResType)) 1874 .addUse(I.getOperand(2).getReg()) 1875 .addUse(I.getOperand(3).getReg()) 1876 .constrainAllUses(TII, TRI, RBI); 1877 1878 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1879 .addDef(ResVReg) 1880 .addUse(GR.getSPIRVTypeID(ResType)) 1881 .addUse(Dot) 1882 .addUse(I.getOperand(4).getReg()) 1883 .constrainAllUses(TII, TRI, RBI); 1884 } 1885 1886 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1887 // extract the elements of the packed inputs, multiply them and add the result 1888 // to the accumulator. 1889 template <bool Signed> 1890 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1891 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1892 assert(I.getNumOperands() == 5); 1893 assert(I.getOperand(2).isReg()); 1894 assert(I.getOperand(3).isReg()); 1895 assert(I.getOperand(4).isReg()); 1896 MachineBasicBlock &BB = *I.getParent(); 1897 1898 bool Result = true; 1899 1900 // Acc = C 1901 Register Acc = I.getOperand(4).getReg(); 1902 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1903 auto ExtractOp = 1904 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1905 1906 // Extract the i8 element, multiply and add it to the accumulator 1907 for (unsigned i = 0; i < 4; i++) { 1908 // A[i] 1909 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1910 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1911 .addDef(AElt) 1912 .addUse(GR.getSPIRVTypeID(ResType)) 1913 .addUse(I.getOperand(2).getReg()) 1914 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1915 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1916 .constrainAllUses(TII, TRI, RBI); 1917 1918 // B[i] 1919 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1920 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1921 .addDef(BElt) 1922 .addUse(GR.getSPIRVTypeID(ResType)) 1923 .addUse(I.getOperand(3).getReg()) 1924 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1925 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1926 .constrainAllUses(TII, TRI, RBI); 1927 1928 // A[i] * B[i] 1929 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1930 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1931 .addDef(Mul) 1932 .addUse(GR.getSPIRVTypeID(ResType)) 1933 .addUse(AElt) 1934 .addUse(BElt) 1935 .constrainAllUses(TII, TRI, RBI); 1936 1937 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1938 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1939 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1940 .addDef(MaskMul) 1941 .addUse(GR.getSPIRVTypeID(ResType)) 1942 .addUse(Mul) 1943 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1944 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1945 .constrainAllUses(TII, TRI, RBI); 1946 1947 // Acc = Acc + A[i] * B[i] 1948 Register Sum = 1949 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1950 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1951 .addDef(Sum) 1952 .addUse(GR.getSPIRVTypeID(ResType)) 1953 .addUse(Acc) 1954 .addUse(MaskMul) 1955 .constrainAllUses(TII, TRI, RBI); 1956 1957 Acc = Sum; 1958 } 1959 1960 return Result; 1961 } 1962 1963 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1964 /// does not have a saturate builtin. 1965 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1966 const SPIRVType *ResType, 1967 MachineInstr &I) const { 1968 assert(I.getNumOperands() == 3); 1969 assert(I.getOperand(2).isReg()); 1970 MachineBasicBlock &BB = *I.getParent(); 1971 Register VZero = buildZerosValF(ResType, I); 1972 Register VOne = buildOnesValF(ResType, I); 1973 1974 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1975 .addDef(ResVReg) 1976 .addUse(GR.getSPIRVTypeID(ResType)) 1977 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1978 .addImm(GL::FClamp) 1979 .addUse(I.getOperand(2).getReg()) 1980 .addUse(VZero) 1981 .addUse(VOne) 1982 .constrainAllUses(TII, TRI, RBI); 1983 } 1984 1985 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1986 const SPIRVType *ResType, 1987 MachineInstr &I) const { 1988 assert(I.getNumOperands() == 3); 1989 assert(I.getOperand(2).isReg()); 1990 MachineBasicBlock &BB = *I.getParent(); 1991 Register InputRegister = I.getOperand(2).getReg(); 1992 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1993 auto &DL = I.getDebugLoc(); 1994 1995 if (!InputType) 1996 report_fatal_error("Input Type could not be determined."); 1997 1998 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1999 2000 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 2001 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 2002 2003 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 2004 2005 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 2006 Register SignReg = NeedsConversion 2007 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 2008 : ResVReg; 2009 2010 bool Result = 2011 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 2012 .addDef(SignReg) 2013 .addUse(GR.getSPIRVTypeID(InputType)) 2014 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2015 .addImm(SignOpcode) 2016 .addUse(InputRegister) 2017 .constrainAllUses(TII, TRI, RBI); 2018 2019 if (NeedsConversion) { 2020 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2021 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2022 .addDef(ResVReg) 2023 .addUse(GR.getSPIRVTypeID(ResType)) 2024 .addUse(SignReg) 2025 .constrainAllUses(TII, TRI, RBI); 2026 } 2027 2028 return Result; 2029 } 2030 2031 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2032 const SPIRVType *ResType, 2033 MachineInstr &I, 2034 unsigned Opcode) const { 2035 MachineBasicBlock &BB = *I.getParent(); 2036 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2037 2038 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2039 .addDef(ResVReg) 2040 .addUse(GR.getSPIRVTypeID(ResType)) 2041 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2042 IntTy, TII)); 2043 2044 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2045 BMI.addUse(I.getOperand(J).getReg()); 2046 } 2047 2048 return BMI.constrainAllUses(TII, TRI, RBI); 2049 } 2050 2051 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2052 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2053 2054 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2055 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2056 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2057 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2058 SPIRV::OpGroupNonUniformBallot); 2059 2060 MachineBasicBlock &BB = *I.getParent(); 2061 Result &= 2062 BuildMI(BB, I, I.getDebugLoc(), 2063 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2064 .addDef(ResVReg) 2065 .addUse(GR.getSPIRVTypeID(ResType)) 2066 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2067 .addImm(SPIRV::GroupOperation::Reduce) 2068 .addUse(BallotReg) 2069 .constrainAllUses(TII, TRI, RBI); 2070 2071 return Result; 2072 } 2073 2074 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2075 const SPIRVType *ResType, 2076 MachineInstr &I) const { 2077 MachineBasicBlock &BB = *I.getParent(); 2078 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2079 .addDef(ResVReg) 2080 .addUse(GR.getSPIRVTypeID(ResType)) 2081 .addUse(I.getOperand(1).getReg()) 2082 .constrainAllUses(TII, TRI, RBI); 2083 } 2084 2085 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2086 const SPIRVType *ResType, 2087 MachineInstr &I) const { 2088 // There is no way to implement `freeze` correctly without support on SPIR-V 2089 // standard side, but we may at least address a simple (static) case when 2090 // undef/poison value presence is obvious. The main benefit of even 2091 // incomplete `freeze` support is preventing of translation from crashing due 2092 // to lack of support on legalization and instruction selection steps. 2093 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2094 return false; 2095 Register OpReg = I.getOperand(1).getReg(); 2096 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2097 Register Reg; 2098 switch (Def->getOpcode()) { 2099 case SPIRV::ASSIGN_TYPE: 2100 if (MachineInstr *AssignToDef = 2101 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2102 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2103 Reg = Def->getOperand(2).getReg(); 2104 } 2105 break; 2106 case SPIRV::OpUndef: 2107 Reg = Def->getOperand(1).getReg(); 2108 break; 2109 } 2110 unsigned DestOpCode; 2111 if (Reg.isValid()) { 2112 DestOpCode = SPIRV::OpConstantNull; 2113 } else { 2114 DestOpCode = TargetOpcode::COPY; 2115 Reg = OpReg; 2116 } 2117 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2118 .addDef(I.getOperand(0).getReg()) 2119 .addUse(Reg) 2120 .constrainAllUses(TII, TRI, RBI); 2121 } 2122 return false; 2123 } 2124 2125 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2126 const SPIRVType *ResType) { 2127 Register OpReg = ResType->getOperand(2).getReg(); 2128 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2129 if (!OpDef) 2130 return 0; 2131 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2132 OpDef->getOperand(1).isReg()) { 2133 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2134 OpDef = RefDef; 2135 } 2136 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2137 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2138 : 0; 2139 return N; 2140 } 2141 2142 // Return true if the type represents a constant register 2143 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2144 SmallPtrSet<SPIRVType *, 4> &Visited) { 2145 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2146 OpDef->getOperand(1).isReg()) { 2147 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2148 OpDef = RefDef; 2149 } 2150 2151 if (Visited.contains(OpDef)) 2152 return true; 2153 Visited.insert(OpDef); 2154 2155 unsigned Opcode = OpDef->getOpcode(); 2156 switch (Opcode) { 2157 case TargetOpcode::G_CONSTANT: 2158 case TargetOpcode::G_FCONSTANT: 2159 return true; 2160 case TargetOpcode::G_INTRINSIC: 2161 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2162 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2163 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2164 Intrinsic::spv_const_composite; 2165 case TargetOpcode::G_BUILD_VECTOR: 2166 case TargetOpcode::G_SPLAT_VECTOR: { 2167 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2168 i++) { 2169 SPIRVType *OpNestedDef = 2170 OpDef->getOperand(i).isReg() 2171 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2172 : nullptr; 2173 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2174 return false; 2175 } 2176 return true; 2177 } 2178 } 2179 return false; 2180 } 2181 2182 // Return true if the virtual register represents a constant 2183 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2184 SmallPtrSet<SPIRVType *, 4> Visited; 2185 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2186 return isConstReg(MRI, OpDef, Visited); 2187 return false; 2188 } 2189 2190 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2191 const SPIRVType *ResType, 2192 MachineInstr &I) const { 2193 unsigned N = 0; 2194 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2195 N = GR.getScalarOrVectorComponentCount(ResType); 2196 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2197 N = getArrayComponentCount(MRI, ResType); 2198 else 2199 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2200 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2201 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2202 2203 // check if we may construct a constant vector 2204 bool IsConst = true; 2205 for (unsigned i = I.getNumExplicitDefs(); 2206 i < I.getNumExplicitOperands() && IsConst; ++i) 2207 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2208 IsConst = false; 2209 2210 if (!IsConst && N < 2) 2211 report_fatal_error( 2212 "There must be at least two constituent operands in a vector"); 2213 2214 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2215 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2216 TII.get(IsConst ? SPIRV::OpConstantComposite 2217 : SPIRV::OpCompositeConstruct)) 2218 .addDef(ResVReg) 2219 .addUse(GR.getSPIRVTypeID(ResType)); 2220 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2221 MIB.addUse(I.getOperand(i).getReg()); 2222 return MIB.constrainAllUses(TII, TRI, RBI); 2223 } 2224 2225 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2226 const SPIRVType *ResType, 2227 MachineInstr &I) const { 2228 unsigned N = 0; 2229 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2230 N = GR.getScalarOrVectorComponentCount(ResType); 2231 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2232 N = getArrayComponentCount(MRI, ResType); 2233 else 2234 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2235 2236 unsigned OpIdx = I.getNumExplicitDefs(); 2237 if (!I.getOperand(OpIdx).isReg()) 2238 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2239 2240 // check if we may construct a constant vector 2241 Register OpReg = I.getOperand(OpIdx).getReg(); 2242 bool IsConst = isConstReg(MRI, OpReg); 2243 2244 if (!IsConst && N < 2) 2245 report_fatal_error( 2246 "There must be at least two constituent operands in a vector"); 2247 2248 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2249 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2250 TII.get(IsConst ? SPIRV::OpConstantComposite 2251 : SPIRV::OpCompositeConstruct)) 2252 .addDef(ResVReg) 2253 .addUse(GR.getSPIRVTypeID(ResType)); 2254 for (unsigned i = 0; i < N; ++i) 2255 MIB.addUse(OpReg); 2256 return MIB.constrainAllUses(TII, TRI, RBI); 2257 } 2258 2259 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2260 const SPIRVType *ResType, 2261 MachineInstr &I) const { 2262 2263 unsigned Opcode; 2264 2265 if (STI.canUseExtension( 2266 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2267 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2268 Opcode = SPIRV::OpDemoteToHelperInvocation; 2269 } else { 2270 Opcode = SPIRV::OpKill; 2271 // OpKill must be the last operation of any basic block. 2272 if (MachineInstr *NextI = I.getNextNode()) { 2273 GR.invalidateMachineInstr(NextI); 2274 NextI->removeFromParent(); 2275 } 2276 } 2277 2278 MachineBasicBlock &BB = *I.getParent(); 2279 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2280 .constrainAllUses(TII, TRI, RBI); 2281 } 2282 2283 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2284 const SPIRVType *ResType, 2285 unsigned CmpOpc, 2286 MachineInstr &I) const { 2287 Register Cmp0 = I.getOperand(2).getReg(); 2288 Register Cmp1 = I.getOperand(3).getReg(); 2289 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2290 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2291 "CMP operands should have the same type"); 2292 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2293 .addDef(ResVReg) 2294 .addUse(GR.getSPIRVTypeID(ResType)) 2295 .addUse(Cmp0) 2296 .addUse(Cmp1) 2297 .constrainAllUses(TII, TRI, RBI); 2298 } 2299 2300 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2301 const SPIRVType *ResType, 2302 MachineInstr &I) const { 2303 auto Pred = I.getOperand(1).getPredicate(); 2304 unsigned CmpOpc; 2305 2306 Register CmpOperand = I.getOperand(2).getReg(); 2307 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2308 CmpOpc = getPtrCmpOpcode(Pred); 2309 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2310 CmpOpc = getBoolCmpOpcode(Pred); 2311 else 2312 CmpOpc = getICmpOpcode(Pred); 2313 return selectCmp(ResVReg, ResType, CmpOpc, I); 2314 } 2315 2316 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2317 const MachineInstr &I, 2318 int OpIdx) const { 2319 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2320 "Expected G_FCONSTANT"); 2321 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2322 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2323 } 2324 2325 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2326 const MachineInstr &I, 2327 int OpIdx) const { 2328 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2329 "Expected G_CONSTANT"); 2330 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2331 } 2332 2333 std::pair<Register, bool> 2334 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2335 const SPIRVType *ResType) const { 2336 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2337 const SPIRVType *SpvI32Ty = 2338 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2339 // Find a constant in DT or build a new one. 2340 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2341 Register NewReg = GR.find(ConstInt, GR.CurMF); 2342 bool Result = true; 2343 if (!NewReg.isValid()) { 2344 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2345 GR.add(ConstInt, GR.CurMF, NewReg); 2346 MachineInstr *MI; 2347 MachineBasicBlock &BB = *I.getParent(); 2348 if (Val == 0) { 2349 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2350 .addDef(NewReg) 2351 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2352 } else { 2353 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2354 .addDef(NewReg) 2355 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2356 .addImm(APInt(32, Val).getZExtValue()); 2357 } 2358 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2359 } 2360 return {NewReg, Result}; 2361 } 2362 2363 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2364 const SPIRVType *ResType, 2365 MachineInstr &I) const { 2366 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2367 return selectCmp(ResVReg, ResType, CmpOp, I); 2368 } 2369 2370 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2371 MachineInstr &I) const { 2372 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2373 bool ZeroAsNull = STI.isOpenCLEnv(); 2374 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2375 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2376 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2377 } 2378 2379 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2380 MachineInstr &I) const { 2381 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2382 bool ZeroAsNull = STI.isOpenCLEnv(); 2383 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2384 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2385 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2386 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2387 } 2388 2389 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2390 MachineInstr &I) const { 2391 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2392 bool ZeroAsNull = STI.isOpenCLEnv(); 2393 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2394 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2395 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2396 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2397 } 2398 2399 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2400 const SPIRVType *ResType, 2401 MachineInstr &I) const { 2402 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2403 APInt One = 2404 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2405 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2406 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2407 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2408 } 2409 2410 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2411 const SPIRVType *ResType, 2412 MachineInstr &I, 2413 bool IsSigned) const { 2414 // To extend a bool, we need to use OpSelect between constants. 2415 Register ZeroReg = buildZerosVal(ResType, I); 2416 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2417 bool IsScalarBool = 2418 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2419 unsigned Opcode = 2420 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2421 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2422 .addDef(ResVReg) 2423 .addUse(GR.getSPIRVTypeID(ResType)) 2424 .addUse(I.getOperand(1).getReg()) 2425 .addUse(OneReg) 2426 .addUse(ZeroReg) 2427 .constrainAllUses(TII, TRI, RBI); 2428 } 2429 2430 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2431 const SPIRVType *ResType, 2432 MachineInstr &I, bool IsSigned, 2433 unsigned Opcode) const { 2434 Register SrcReg = I.getOperand(1).getReg(); 2435 // We can convert bool value directly to float type without OpConvert*ToF, 2436 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2437 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2438 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2439 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2440 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2441 const unsigned NumElts = ResType->getOperand(2).getImm(); 2442 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2443 } 2444 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2445 selectSelect(SrcReg, TmpType, I, false); 2446 } 2447 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2448 } 2449 2450 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2451 const SPIRVType *ResType, 2452 MachineInstr &I, bool IsSigned) const { 2453 Register SrcReg = I.getOperand(1).getReg(); 2454 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2455 return selectSelect(ResVReg, ResType, I, IsSigned); 2456 2457 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2458 if (SrcType == ResType) 2459 return BuildCOPY(ResVReg, SrcReg, I); 2460 2461 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2462 return selectUnOp(ResVReg, ResType, I, Opcode); 2463 } 2464 2465 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2466 const SPIRVType *ResType, 2467 MachineInstr &I, 2468 bool IsSigned) const { 2469 MachineIRBuilder MIRBuilder(I); 2470 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2471 MachineBasicBlock &BB = *I.getParent(); 2472 // Ensure we have bool. 2473 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2474 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2475 if (N > 1) 2476 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2477 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2478 // Build less-than-equal and less-than. 2479 // TODO: replace with one-liner createVirtualRegister() from 2480 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2481 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2482 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2483 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2484 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2485 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2486 : SPIRV::OpULessThanEqual)) 2487 .addDef(IsLessEqReg) 2488 .addUse(BoolTypeReg) 2489 .addUse(I.getOperand(1).getReg()) 2490 .addUse(I.getOperand(2).getReg()) 2491 .constrainAllUses(TII, TRI, RBI); 2492 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2493 MRI->setType(IsLessReg, LLT::scalar(64)); 2494 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2495 Result &= BuildMI(BB, I, I.getDebugLoc(), 2496 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2497 .addDef(IsLessReg) 2498 .addUse(BoolTypeReg) 2499 .addUse(I.getOperand(1).getReg()) 2500 .addUse(I.getOperand(2).getReg()) 2501 .constrainAllUses(TII, TRI, RBI); 2502 // Build selects. 2503 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2504 Register NegOneOrZeroReg = 2505 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2506 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2507 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2508 unsigned SelectOpcode = 2509 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2510 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2511 .addDef(NegOneOrZeroReg) 2512 .addUse(ResTypeReg) 2513 .addUse(IsLessReg) 2514 .addUse(buildOnesVal(true, ResType, I)) // -1 2515 .addUse(buildZerosVal(ResType, I)) 2516 .constrainAllUses(TII, TRI, RBI); 2517 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2518 .addDef(ResVReg) 2519 .addUse(ResTypeReg) 2520 .addUse(IsLessEqReg) 2521 .addUse(NegOneOrZeroReg) // -1 or 0 2522 .addUse(buildOnesVal(false, ResType, I)) 2523 .constrainAllUses(TII, TRI, RBI); 2524 } 2525 2526 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2527 Register ResVReg, 2528 MachineInstr &I, 2529 const SPIRVType *IntTy, 2530 const SPIRVType *BoolTy) const { 2531 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2532 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2533 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2534 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2535 Register Zero = buildZerosVal(IntTy, I); 2536 Register One = buildOnesVal(false, IntTy, I); 2537 MachineBasicBlock &BB = *I.getParent(); 2538 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2539 .addDef(BitIntReg) 2540 .addUse(GR.getSPIRVTypeID(IntTy)) 2541 .addUse(IntReg) 2542 .addUse(One) 2543 .constrainAllUses(TII, TRI, RBI); 2544 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2545 .addDef(ResVReg) 2546 .addUse(GR.getSPIRVTypeID(BoolTy)) 2547 .addUse(BitIntReg) 2548 .addUse(Zero) 2549 .constrainAllUses(TII, TRI, RBI); 2550 } 2551 2552 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2553 const SPIRVType *ResType, 2554 MachineInstr &I) const { 2555 Register IntReg = I.getOperand(1).getReg(); 2556 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2557 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2558 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2559 if (ArgType == ResType) 2560 return BuildCOPY(ResVReg, IntReg, I); 2561 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2562 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2563 return selectUnOp(ResVReg, ResType, I, Opcode); 2564 } 2565 2566 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2567 const SPIRVType *ResType, 2568 const APInt &Imm, 2569 MachineInstr &I) const { 2570 unsigned TyOpcode = ResType->getOpcode(); 2571 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2572 MachineBasicBlock &BB = *I.getParent(); 2573 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2574 Imm.isZero()) 2575 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2576 .addDef(ResVReg) 2577 .addUse(GR.getSPIRVTypeID(ResType)) 2578 .constrainAllUses(TII, TRI, RBI); 2579 if (TyOpcode == SPIRV::OpTypeInt) { 2580 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2581 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2582 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2583 } 2584 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2585 .addDef(ResVReg) 2586 .addUse(GR.getSPIRVTypeID(ResType)); 2587 // <=32-bit integers should be caught by the sdag pattern. 2588 assert(Imm.getBitWidth() > 32); 2589 addNumImm(Imm, MIB); 2590 return MIB.constrainAllUses(TII, TRI, RBI); 2591 } 2592 2593 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2594 const SPIRVType *ResType, 2595 MachineInstr &I) const { 2596 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2597 .addDef(ResVReg) 2598 .addUse(GR.getSPIRVTypeID(ResType)) 2599 .constrainAllUses(TII, TRI, RBI); 2600 } 2601 2602 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2603 assert(MO.isReg()); 2604 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2605 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2606 assert(TypeInst->getOperand(1).isReg()); 2607 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2608 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2609 } 2610 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2611 } 2612 2613 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2614 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2615 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2616 return TypeInst->getOperand(2).getImm(); 2617 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2618 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2619 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2620 } 2621 2622 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2623 const SPIRVType *ResType, 2624 MachineInstr &I) const { 2625 MachineBasicBlock &BB = *I.getParent(); 2626 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2627 .addDef(ResVReg) 2628 .addUse(GR.getSPIRVTypeID(ResType)) 2629 // object to insert 2630 .addUse(I.getOperand(3).getReg()) 2631 // composite to insert into 2632 .addUse(I.getOperand(2).getReg()); 2633 for (unsigned i = 4; i < I.getNumOperands(); i++) 2634 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2635 return MIB.constrainAllUses(TII, TRI, RBI); 2636 } 2637 2638 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2639 const SPIRVType *ResType, 2640 MachineInstr &I) const { 2641 MachineBasicBlock &BB = *I.getParent(); 2642 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2643 .addDef(ResVReg) 2644 .addUse(GR.getSPIRVTypeID(ResType)) 2645 .addUse(I.getOperand(2).getReg()); 2646 for (unsigned i = 3; i < I.getNumOperands(); i++) 2647 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2648 return MIB.constrainAllUses(TII, TRI, RBI); 2649 } 2650 2651 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2652 const SPIRVType *ResType, 2653 MachineInstr &I) const { 2654 if (isImm(I.getOperand(4), MRI)) 2655 return selectInsertVal(ResVReg, ResType, I); 2656 MachineBasicBlock &BB = *I.getParent(); 2657 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2658 .addDef(ResVReg) 2659 .addUse(GR.getSPIRVTypeID(ResType)) 2660 .addUse(I.getOperand(2).getReg()) 2661 .addUse(I.getOperand(3).getReg()) 2662 .addUse(I.getOperand(4).getReg()) 2663 .constrainAllUses(TII, TRI, RBI); 2664 } 2665 2666 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2667 const SPIRVType *ResType, 2668 MachineInstr &I) const { 2669 if (isImm(I.getOperand(3), MRI)) 2670 return selectExtractVal(ResVReg, ResType, I); 2671 MachineBasicBlock &BB = *I.getParent(); 2672 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2673 .addDef(ResVReg) 2674 .addUse(GR.getSPIRVTypeID(ResType)) 2675 .addUse(I.getOperand(2).getReg()) 2676 .addUse(I.getOperand(3).getReg()) 2677 .constrainAllUses(TII, TRI, RBI); 2678 } 2679 2680 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2681 const SPIRVType *ResType, 2682 MachineInstr &I) const { 2683 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2684 2685 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2686 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2687 // we have to use Op[InBounds]AccessChain. 2688 const unsigned Opcode = STI.isVulkanEnv() 2689 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2690 : SPIRV::OpAccessChain) 2691 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2692 : SPIRV::OpPtrAccessChain); 2693 2694 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2695 .addDef(ResVReg) 2696 .addUse(GR.getSPIRVTypeID(ResType)) 2697 // Object to get a pointer to. 2698 .addUse(I.getOperand(3).getReg()); 2699 // Adding indices. 2700 const unsigned StartingIndex = 2701 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2702 ? 5 2703 : 4; 2704 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2705 Res.addUse(I.getOperand(i).getReg()); 2706 return Res.constrainAllUses(TII, TRI, RBI); 2707 } 2708 2709 // Maybe wrap a value into OpSpecConstantOp 2710 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2711 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2712 bool Result = true; 2713 unsigned Lim = I.getNumExplicitOperands(); 2714 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2715 Register OpReg = I.getOperand(i).getReg(); 2716 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2717 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2718 SmallPtrSet<SPIRVType *, 4> Visited; 2719 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2720 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2721 GR.isAggregateType(OpType)) { 2722 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2723 // by selectAddrSpaceCast() 2724 CompositeArgs.push_back(OpReg); 2725 continue; 2726 } 2727 MachineFunction *MF = I.getMF(); 2728 Register WrapReg = GR.find(OpDefine, MF); 2729 if (WrapReg.isValid()) { 2730 CompositeArgs.push_back(WrapReg); 2731 continue; 2732 } 2733 // Create a new register for the wrapper 2734 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2735 GR.add(OpDefine, MF, WrapReg); 2736 CompositeArgs.push_back(WrapReg); 2737 // Decorate the wrapper register and generate a new instruction 2738 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2739 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2740 MachineBasicBlock &BB = *I.getParent(); 2741 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2742 .addDef(WrapReg) 2743 .addUse(GR.getSPIRVTypeID(OpType)) 2744 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2745 .addUse(OpReg) 2746 .constrainAllUses(TII, TRI, RBI); 2747 if (!Result) 2748 break; 2749 } 2750 return Result; 2751 } 2752 2753 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2754 const SPIRVType *ResType, 2755 MachineInstr &I) const { 2756 MachineBasicBlock &BB = *I.getParent(); 2757 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2758 switch (IID) { 2759 case Intrinsic::spv_load: 2760 return selectLoad(ResVReg, ResType, I); 2761 case Intrinsic::spv_store: 2762 return selectStore(I); 2763 case Intrinsic::spv_extractv: 2764 return selectExtractVal(ResVReg, ResType, I); 2765 case Intrinsic::spv_insertv: 2766 return selectInsertVal(ResVReg, ResType, I); 2767 case Intrinsic::spv_extractelt: 2768 return selectExtractElt(ResVReg, ResType, I); 2769 case Intrinsic::spv_insertelt: 2770 return selectInsertElt(ResVReg, ResType, I); 2771 case Intrinsic::spv_gep: 2772 return selectGEP(ResVReg, ResType, I); 2773 case Intrinsic::spv_unref_global: 2774 case Intrinsic::spv_init_global: { 2775 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2776 MachineInstr *Init = I.getNumExplicitOperands() > 2 2777 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2778 : nullptr; 2779 assert(MI); 2780 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2781 } 2782 case Intrinsic::spv_undef: { 2783 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2784 .addDef(ResVReg) 2785 .addUse(GR.getSPIRVTypeID(ResType)); 2786 return MIB.constrainAllUses(TII, TRI, RBI); 2787 } 2788 case Intrinsic::spv_const_composite: { 2789 // If no values are attached, the composite is null constant. 2790 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2791 // Select a proper instruction. 2792 unsigned Opcode = SPIRV::OpConstantNull; 2793 SmallVector<Register> CompositeArgs; 2794 if (!IsNull) { 2795 Opcode = SPIRV::OpConstantComposite; 2796 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2797 return false; 2798 } 2799 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2800 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2801 .addDef(ResVReg) 2802 .addUse(GR.getSPIRVTypeID(ResType)); 2803 // skip type MD node we already used when generated assign.type for this 2804 if (!IsNull) { 2805 for (Register OpReg : CompositeArgs) 2806 MIB.addUse(OpReg); 2807 } 2808 return MIB.constrainAllUses(TII, TRI, RBI); 2809 } 2810 case Intrinsic::spv_assign_name: { 2811 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2812 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2813 for (unsigned i = I.getNumExplicitDefs() + 2; 2814 i < I.getNumExplicitOperands(); ++i) { 2815 MIB.addImm(I.getOperand(i).getImm()); 2816 } 2817 return MIB.constrainAllUses(TII, TRI, RBI); 2818 } 2819 case Intrinsic::spv_switch: { 2820 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2821 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2822 if (I.getOperand(i).isReg()) 2823 MIB.addReg(I.getOperand(i).getReg()); 2824 else if (I.getOperand(i).isCImm()) 2825 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2826 else if (I.getOperand(i).isMBB()) 2827 MIB.addMBB(I.getOperand(i).getMBB()); 2828 else 2829 llvm_unreachable("Unexpected OpSwitch operand"); 2830 } 2831 return MIB.constrainAllUses(TII, TRI, RBI); 2832 } 2833 case Intrinsic::spv_loop_merge: { 2834 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge)); 2835 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2836 assert(I.getOperand(i).isMBB()); 2837 MIB.addMBB(I.getOperand(i).getMBB()); 2838 } 2839 MIB.addImm(SPIRV::SelectionControl::None); 2840 return MIB.constrainAllUses(TII, TRI, RBI); 2841 } 2842 case Intrinsic::spv_selection_merge: { 2843 auto MIB = 2844 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge)); 2845 assert(I.getOperand(1).isMBB() && 2846 "operand 1 to spv_selection_merge must be a basic block"); 2847 MIB.addMBB(I.getOperand(1).getMBB()); 2848 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm())); 2849 return MIB.constrainAllUses(TII, TRI, RBI); 2850 } 2851 case Intrinsic::spv_cmpxchg: 2852 return selectAtomicCmpXchg(ResVReg, ResType, I); 2853 case Intrinsic::spv_unreachable: 2854 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2855 .constrainAllUses(TII, TRI, RBI); 2856 case Intrinsic::spv_alloca: 2857 return selectFrameIndex(ResVReg, ResType, I); 2858 case Intrinsic::spv_alloca_array: 2859 return selectAllocaArray(ResVReg, ResType, I); 2860 case Intrinsic::spv_assume: 2861 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2862 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2863 .addUse(I.getOperand(1).getReg()) 2864 .constrainAllUses(TII, TRI, RBI); 2865 break; 2866 case Intrinsic::spv_expect: 2867 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2868 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2869 .addDef(ResVReg) 2870 .addUse(GR.getSPIRVTypeID(ResType)) 2871 .addUse(I.getOperand(2).getReg()) 2872 .addUse(I.getOperand(3).getReg()) 2873 .constrainAllUses(TII, TRI, RBI); 2874 break; 2875 case Intrinsic::arithmetic_fence: 2876 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2877 return BuildMI(BB, I, I.getDebugLoc(), 2878 TII.get(SPIRV::OpArithmeticFenceEXT)) 2879 .addDef(ResVReg) 2880 .addUse(GR.getSPIRVTypeID(ResType)) 2881 .addUse(I.getOperand(2).getReg()) 2882 .constrainAllUses(TII, TRI, RBI); 2883 else 2884 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2885 break; 2886 case Intrinsic::spv_thread_id: 2887 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 2888 // intrinsic in LLVM IR for SPIR-V backend. 2889 // 2890 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 2891 // `GlobalInvocationId` builtin variable 2892 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 2893 ResType, I); 2894 case Intrinsic::spv_thread_id_in_group: 2895 // The HLSL SV_GroupThreadId semantic is lowered to 2896 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 2897 // 2898 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 2899 // translated to a `LocalInvocationId` builtin variable 2900 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 2901 ResType, I); 2902 case Intrinsic::spv_group_id: 2903 // The HLSL SV_GroupId semantic is lowered to 2904 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. 2905 // 2906 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` 2907 // builtin variable 2908 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, 2909 I); 2910 case Intrinsic::spv_fdot: 2911 return selectFloatDot(ResVReg, ResType, I); 2912 case Intrinsic::spv_udot: 2913 case Intrinsic::spv_sdot: 2914 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2915 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2916 return selectIntegerDot(ResVReg, ResType, I, 2917 /*Signed=*/IID == Intrinsic::spv_sdot); 2918 return selectIntegerDotExpansion(ResVReg, ResType, I); 2919 case Intrinsic::spv_dot4add_i8packed: 2920 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2921 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2922 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2923 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2924 case Intrinsic::spv_dot4add_u8packed: 2925 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2926 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2927 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2928 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2929 case Intrinsic::spv_all: 2930 return selectAll(ResVReg, ResType, I); 2931 case Intrinsic::spv_any: 2932 return selectAny(ResVReg, ResType, I); 2933 case Intrinsic::spv_cross: 2934 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2935 case Intrinsic::spv_distance: 2936 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); 2937 case Intrinsic::spv_lerp: 2938 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2939 case Intrinsic::spv_length: 2940 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2941 case Intrinsic::spv_degrees: 2942 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2943 case Intrinsic::spv_frac: 2944 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2945 case Intrinsic::spv_normalize: 2946 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2947 case Intrinsic::spv_rsqrt: 2948 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2949 case Intrinsic::spv_sign: 2950 return selectSign(ResVReg, ResType, I); 2951 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2952 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2953 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2954 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2955 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2956 bool Result = true; 2957 auto MemSemConstant = 2958 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2959 Register MemSemReg = MemSemConstant.first; 2960 Result &= MemSemConstant.second; 2961 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2962 Register ScopeReg = ScopeConstant.first; 2963 Result &= ScopeConstant.second; 2964 MachineBasicBlock &BB = *I.getParent(); 2965 return Result && 2966 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2967 .addUse(ScopeReg) 2968 .addUse(ScopeReg) 2969 .addUse(MemSemReg) 2970 .constrainAllUses(TII, TRI, RBI); 2971 } 2972 case Intrinsic::spv_lifetime_start: 2973 case Intrinsic::spv_lifetime_end: { 2974 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2975 : SPIRV::OpLifetimeStop; 2976 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2977 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2978 if (Size == -1) 2979 Size = 0; 2980 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2981 .addUse(PtrReg) 2982 .addImm(Size) 2983 .constrainAllUses(TII, TRI, RBI); 2984 } 2985 case Intrinsic::spv_saturate: 2986 return selectSaturate(ResVReg, ResType, I); 2987 case Intrinsic::spv_nclamp: 2988 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2989 case Intrinsic::spv_uclamp: 2990 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2991 case Intrinsic::spv_sclamp: 2992 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2993 case Intrinsic::spv_wave_active_countbits: 2994 return selectWaveActiveCountBits(ResVReg, ResType, I); 2995 case Intrinsic::spv_wave_all: 2996 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 2997 case Intrinsic::spv_wave_any: 2998 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 2999 case Intrinsic::spv_wave_is_first_lane: 3000 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 3001 case Intrinsic::spv_wave_readlane: 3002 return selectWaveOpInst(ResVReg, ResType, I, 3003 SPIRV::OpGroupNonUniformShuffle); 3004 case Intrinsic::spv_step: 3005 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 3006 case Intrinsic::spv_radians: 3007 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 3008 // Discard intrinsics which we do not expect to actually represent code after 3009 // lowering or intrinsics which are not implemented but should not crash when 3010 // found in a customer's LLVM IR input. 3011 case Intrinsic::instrprof_increment: 3012 case Intrinsic::instrprof_increment_step: 3013 case Intrinsic::instrprof_value_profile: 3014 break; 3015 // Discard internal intrinsics. 3016 case Intrinsic::spv_value_md: 3017 break; 3018 case Intrinsic::spv_resource_handlefrombinding: { 3019 return selectHandleFromBinding(ResVReg, ResType, I); 3020 } 3021 case Intrinsic::spv_resource_store_typedbuffer: { 3022 return selectImageWriteIntrinsic(I); 3023 } 3024 case Intrinsic::spv_resource_load_typedbuffer: { 3025 return selectReadImageIntrinsic(ResVReg, ResType, I); 3026 } 3027 case Intrinsic::spv_discard: { 3028 return selectDiscard(ResVReg, ResType, I); 3029 } 3030 default: { 3031 std::string DiagMsg; 3032 raw_string_ostream OS(DiagMsg); 3033 I.print(OS); 3034 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3035 report_fatal_error(DiagMsg.c_str(), false); 3036 } 3037 } 3038 return true; 3039 } 3040 3041 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3042 const SPIRVType *ResType, 3043 MachineInstr &I) const { 3044 3045 uint32_t Set = foldImm(I.getOperand(2), MRI); 3046 uint32_t Binding = foldImm(I.getOperand(3), MRI); 3047 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 3048 Register IndexReg = I.getOperand(5).getReg(); 3049 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 3050 3051 MachineIRBuilder MIRBuilder(I); 3052 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3053 IndexReg, IsNonUniform, MIRBuilder); 3054 3055 if (IsNonUniform) 3056 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 3057 3058 // TODO: For now we assume the resource is an image, which needs to be 3059 // loaded to get the handle. That will not be true for storage buffers. 3060 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3061 .addDef(ResVReg) 3062 .addUse(GR.getSPIRVTypeID(ResType)) 3063 .addUse(VarReg) 3064 .constrainAllUses(TII, TRI, RBI); 3065 } 3066 3067 bool SPIRVInstructionSelector::selectReadImageIntrinsic( 3068 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3069 3070 // If the load of the image is in a different basic block, then 3071 // this will generate invalid code. A proper solution is to move 3072 // the OpLoad from selectHandleFromBinding here. However, to do 3073 // that we will need to change the return type of the intrinsic. 3074 // We will do that when we can, but for now trying to move forward with other 3075 // issues. 3076 Register ImageReg = I.getOperand(2).getReg(); 3077 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3078 "The image must be loaded in the same basic block as its use."); 3079 3080 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3081 if (ResultSize == 4) { 3082 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3083 TII.get(SPIRV::OpImageRead)) 3084 .addDef(ResVReg) 3085 .addUse(GR.getSPIRVTypeID(ResType)) 3086 .addUse(ImageReg) 3087 .addUse(I.getOperand(3).getReg()) 3088 .constrainAllUses(TII, TRI, RBI); 3089 } 3090 3091 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 3092 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3093 bool Succeed = 3094 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3095 .addDef(ReadReg) 3096 .addUse(GR.getSPIRVTypeID(ReadType)) 3097 .addUse(ImageReg) 3098 .addUse(I.getOperand(3).getReg()) 3099 .constrainAllUses(TII, TRI, RBI); 3100 if (!Succeed) 3101 return false; 3102 3103 if (ResultSize == 1) { 3104 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3105 TII.get(SPIRV::OpCompositeExtract)) 3106 .addDef(ResVReg) 3107 .addUse(GR.getSPIRVTypeID(ResType)) 3108 .addUse(ReadReg) 3109 .addImm(0) 3110 .constrainAllUses(TII, TRI, RBI); 3111 } 3112 return extractSubvector(ResVReg, ResType, ReadReg, I); 3113 } 3114 3115 bool SPIRVInstructionSelector::extractSubvector( 3116 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3117 MachineInstr &InsertionPoint) const { 3118 SPIRVType *InputType = GR.getResultType(ReadReg); 3119 [[maybe_unused]] uint64_t InputSize = 3120 GR.getScalarOrVectorComponentCount(InputType); 3121 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3122 assert(InputSize > 1 && "The input must be a vector."); 3123 assert(ResultSize > 1 && "The result must be a vector."); 3124 assert(ResultSize < InputSize && 3125 "Cannot extract more element than there are in the input."); 3126 SmallVector<Register> ComponentRegisters; 3127 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3128 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3129 for (uint64_t I = 0; I < ResultSize; I++) { 3130 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3131 bool Succeed = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3132 InsertionPoint.getDebugLoc(), 3133 TII.get(SPIRV::OpCompositeExtract)) 3134 .addDef(ComponentReg) 3135 .addUse(ScalarType->getOperand(0).getReg()) 3136 .addUse(ReadReg) 3137 .addImm(I) 3138 .constrainAllUses(TII, TRI, RBI); 3139 if (!Succeed) 3140 return false; 3141 ComponentRegisters.emplace_back(ComponentReg); 3142 } 3143 3144 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3145 InsertionPoint.getDebugLoc(), 3146 TII.get(SPIRV::OpCompositeConstruct)) 3147 .addDef(ResVReg) 3148 .addUse(GR.getSPIRVTypeID(ResType)); 3149 3150 for (Register ComponentReg : ComponentRegisters) 3151 MIB.addUse(ComponentReg); 3152 return MIB.constrainAllUses(TII, TRI, RBI); 3153 } 3154 3155 bool SPIRVInstructionSelector::selectImageWriteIntrinsic( 3156 MachineInstr &I) const { 3157 // If the load of the image is in a different basic block, then 3158 // this will generate invalid code. A proper solution is to move 3159 // the OpLoad from selectHandleFromBinding here. However, to do 3160 // that we will need to change the return type of the intrinsic. 3161 // We will do that when we can, but for now trying to move forward with other 3162 // issues. 3163 Register ImageReg = I.getOperand(1).getReg(); 3164 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3165 "The image must be loaded in the same basic block as its use."); 3166 Register CoordinateReg = I.getOperand(2).getReg(); 3167 Register DataReg = I.getOperand(3).getReg(); 3168 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3169 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3170 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3171 TII.get(SPIRV::OpImageWrite)) 3172 .addUse(ImageReg) 3173 .addUse(CoordinateReg) 3174 .addUse(DataReg) 3175 .constrainAllUses(TII, TRI, RBI); 3176 } 3177 3178 Register SPIRVInstructionSelector::buildPointerToResource( 3179 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3180 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3181 MachineIRBuilder MIRBuilder) const { 3182 if (ArraySize == 1) 3183 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3184 MIRBuilder); 3185 3186 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3187 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3188 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3189 VarType, Set, Binding, MIRBuilder); 3190 3191 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3192 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3193 3194 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3195 if (IsNonUniform) { 3196 // It is unclear which value needs to be marked an non-uniform, so both 3197 // the index and the access changed are decorated as non-uniform. 3198 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3199 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3200 } 3201 3202 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3203 .addDef(AcReg) 3204 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3205 .addUse(VarReg) 3206 .addUse(IndexReg); 3207 3208 return AcReg; 3209 } 3210 3211 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3212 const SPIRVType *ResType, 3213 MachineInstr &I, 3214 bool IsSigned) const { 3215 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3216 // zero or sign extend 3217 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3218 bool Result = 3219 selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, Opcode); 3220 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3221 } 3222 3223 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3224 const SPIRVType *ResType, 3225 MachineInstr &I, 3226 Register SrcReg, 3227 bool IsSigned) const { 3228 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3229 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3230 .addDef(ResVReg) 3231 .addUse(GR.getSPIRVTypeID(ResType)) 3232 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3233 .addImm(Opcode) 3234 .addUse(SrcReg) 3235 .constrainAllUses(TII, TRI, RBI); 3236 } 3237 3238 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3239 const SPIRVType *ResType, 3240 MachineInstr &I, 3241 bool IsSigned) const { 3242 Register OpReg = I.getOperand(2).getReg(); 3243 // 1. split our int64 into 2 pieces using a bitcast 3244 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3245 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3246 MachineIRBuilder MIRBuilder(I); 3247 SPIRVType *postCastT = 3248 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3249 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3250 bool Result = 3251 selectOpWithSrcs(bitcastReg, postCastT, I, {OpReg}, SPIRV::OpBitcast); 3252 3253 // 2. call firstbithigh 3254 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3255 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3256 3257 // 3. split result vector into high bits and low bits 3258 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3259 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3260 3261 bool ZeroAsNull = STI.isOpenCLEnv(); 3262 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3263 if (isScalarRes) { 3264 // if scalar do a vector extract 3265 Result &= selectOpWithSrcs( 3266 HighReg, ResType, I, 3267 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3268 SPIRV::OpVectorExtractDynamic); 3269 Result &= selectOpWithSrcs( 3270 LowReg, ResType, I, 3271 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3272 SPIRV::OpVectorExtractDynamic); 3273 } else { // vector case do a shufflevector 3274 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3275 TII.get(SPIRV::OpVectorShuffle)) 3276 .addDef(HighReg) 3277 .addUse(GR.getSPIRVTypeID(ResType)) 3278 .addUse(FBHReg) 3279 .addUse(FBHReg); 3280 // ^^ this vector will not be selected from; could be empty 3281 unsigned j; 3282 for (j = 0; j < count * 2; j += 2) { 3283 MIB.addImm(j); 3284 } 3285 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3286 3287 // get low bits 3288 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3289 TII.get(SPIRV::OpVectorShuffle)) 3290 .addDef(LowReg) 3291 .addUse(GR.getSPIRVTypeID(ResType)) 3292 .addUse(FBHReg) 3293 .addUse(FBHReg); 3294 // ^^ this vector will not be selected from; could be empty 3295 for (j = 1; j < count * 2; j += 2) { 3296 MIB.addImm(j); 3297 } 3298 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3299 } 3300 3301 // 4. check if result of each top 32 bits is == -1 3302 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3303 Register NegOneReg; 3304 Register Reg0; 3305 Register Reg32; 3306 unsigned selectOp; 3307 unsigned addOp; 3308 if (isScalarRes) { 3309 NegOneReg = 3310 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3311 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3312 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3313 selectOp = SPIRV::OpSelectSISCond; 3314 addOp = SPIRV::OpIAddS; 3315 } else { 3316 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3317 NegOneReg = 3318 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3319 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3320 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3321 selectOp = SPIRV::OpSelectVIVCond; 3322 addOp = SPIRV::OpIAddV; 3323 } 3324 3325 // check if the high bits are == -1; true if -1 3326 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3327 Result &= selectOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3328 SPIRV::OpIEqual); 3329 3330 // Select low bits if true in BReg, otherwise high bits 3331 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3332 Result &= 3333 selectOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, selectOp); 3334 3335 // Add 32 for high bits, 0 for low bits 3336 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3337 Result &= selectOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3338 3339 return Result && 3340 selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3341 } 3342 3343 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3344 const SPIRVType *ResType, 3345 MachineInstr &I, 3346 bool IsSigned) const { 3347 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3348 Register OpReg = I.getOperand(2).getReg(); 3349 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3350 3351 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3352 case 16: 3353 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3354 case 32: 3355 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3356 case 64: 3357 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3358 default: 3359 report_fatal_error( 3360 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3361 } 3362 } 3363 3364 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3365 const SPIRVType *ResType, 3366 MachineInstr &I) const { 3367 // there was an allocation size parameter to the allocation instruction 3368 // that is not 1 3369 MachineBasicBlock &BB = *I.getParent(); 3370 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3371 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3372 .addDef(ResVReg) 3373 .addUse(GR.getSPIRVTypeID(ResType)) 3374 .addUse(I.getOperand(2).getReg()) 3375 .constrainAllUses(TII, TRI, RBI); 3376 if (!STI.isVulkanEnv()) { 3377 unsigned Alignment = I.getOperand(3).getImm(); 3378 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3379 } 3380 return Res; 3381 } 3382 3383 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3384 const SPIRVType *ResType, 3385 MachineInstr &I) const { 3386 // Change order of instructions if needed: all OpVariable instructions in a 3387 // function must be the first instructions in the first block 3388 auto It = getOpVariableMBBIt(I); 3389 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3390 TII.get(SPIRV::OpVariable)) 3391 .addDef(ResVReg) 3392 .addUse(GR.getSPIRVTypeID(ResType)) 3393 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3394 .constrainAllUses(TII, TRI, RBI); 3395 if (!STI.isVulkanEnv()) { 3396 unsigned Alignment = I.getOperand(2).getImm(); 3397 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3398 {Alignment}); 3399 } 3400 return Res; 3401 } 3402 3403 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3404 // InstructionSelector walks backwards through the instructions. We can use 3405 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3406 // first, so can generate an OpBranchConditional here. If there is no 3407 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3408 const MachineInstr *PrevI = I.getPrevNode(); 3409 MachineBasicBlock &MBB = *I.getParent(); 3410 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3411 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3412 .addUse(PrevI->getOperand(0).getReg()) 3413 .addMBB(PrevI->getOperand(1).getMBB()) 3414 .addMBB(I.getOperand(0).getMBB()) 3415 .constrainAllUses(TII, TRI, RBI); 3416 } 3417 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3418 .addMBB(I.getOperand(0).getMBB()) 3419 .constrainAllUses(TII, TRI, RBI); 3420 } 3421 3422 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3423 // InstructionSelector walks backwards through the instructions. For an 3424 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3425 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3426 // generate the OpBranchConditional in selectBranch above. 3427 // 3428 // If an OpBranchConditional has been generated, we simply return, as the work 3429 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3430 // implicit fallthrough to the next basic block, so we need to create an 3431 // OpBranchConditional with an explicit "false" argument pointing to the next 3432 // basic block that LLVM would fall through to. 3433 const MachineInstr *NextI = I.getNextNode(); 3434 // Check if this has already been successfully selected. 3435 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3436 return true; 3437 // Must be relying on implicit block fallthrough, so generate an 3438 // OpBranchConditional with the "next" basic block as the "false" target. 3439 MachineBasicBlock &MBB = *I.getParent(); 3440 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3441 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3442 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3443 .addUse(I.getOperand(0).getReg()) 3444 .addMBB(I.getOperand(1).getMBB()) 3445 .addMBB(NextMBB) 3446 .constrainAllUses(TII, TRI, RBI); 3447 } 3448 3449 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3450 const SPIRVType *ResType, 3451 MachineInstr &I) const { 3452 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3453 .addDef(ResVReg) 3454 .addUse(GR.getSPIRVTypeID(ResType)); 3455 const unsigned NumOps = I.getNumOperands(); 3456 for (unsigned i = 1; i < NumOps; i += 2) { 3457 MIB.addUse(I.getOperand(i + 0).getReg()); 3458 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3459 } 3460 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3461 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3462 MIB->removeOperand(1); 3463 return Res; 3464 } 3465 3466 bool SPIRVInstructionSelector::selectGlobalValue( 3467 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3468 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3469 MachineIRBuilder MIRBuilder(I); 3470 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3471 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3472 SPIRVType *PointerBaseType; 3473 if (GVType->isArrayTy()) { 3474 SPIRVType *ArrayElementType = 3475 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3476 SPIRV::AccessQualifier::ReadWrite, false); 3477 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3478 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3479 } else { 3480 PointerBaseType = GR.getOrCreateSPIRVType( 3481 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3482 } 3483 3484 std::string GlobalIdent; 3485 if (!GV->hasName()) { 3486 unsigned &ID = UnnamedGlobalIDs[GV]; 3487 if (ID == 0) 3488 ID = UnnamedGlobalIDs.size(); 3489 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3490 } else { 3491 GlobalIdent = GV->getName(); 3492 } 3493 3494 // Behaviour of functions as operands depends on availability of the 3495 // corresponding extension (SPV_INTEL_function_pointers): 3496 // - If there is an extension to operate with functions as operands: 3497 // We create a proper constant operand and evaluate a correct type for a 3498 // function pointer. 3499 // - Without the required extension: 3500 // We have functions as operands in tests with blocks of instruction e.g. in 3501 // transcoding/global_block.ll. These operands are not used and should be 3502 // substituted by zero constants. Their type is expected to be always 3503 // OpTypePointer Function %uchar. 3504 if (isa<Function>(GV)) { 3505 const Constant *ConstVal = GV; 3506 MachineBasicBlock &BB = *I.getParent(); 3507 Register NewReg = GR.find(ConstVal, GR.CurMF); 3508 if (!NewReg.isValid()) { 3509 Register NewReg = ResVReg; 3510 GR.add(ConstVal, GR.CurMF, NewReg); 3511 const Function *GVFun = 3512 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3513 ? dyn_cast<Function>(GV) 3514 : nullptr; 3515 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3516 PointerBaseType, I, TII, 3517 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3518 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3519 if (GVFun) { 3520 // References to a function via function pointers generate virtual 3521 // registers without a definition. We will resolve it later, during 3522 // module analysis stage. 3523 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 3524 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3525 Register FuncVReg = 3526 MRI->createGenericVirtualRegister(GR.getRegType(ResType)); 3527 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); 3528 MachineInstrBuilder MIB1 = 3529 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 3530 .addDef(FuncVReg) 3531 .addUse(ResTypeReg); 3532 MachineInstrBuilder MIB2 = 3533 BuildMI(BB, I, I.getDebugLoc(), 3534 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3535 .addDef(NewReg) 3536 .addUse(ResTypeReg) 3537 .addUse(FuncVReg); 3538 // mapping the function pointer to the used Function 3539 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); 3540 return MIB1.constrainAllUses(TII, TRI, RBI) && 3541 MIB2.constrainAllUses(TII, TRI, RBI); 3542 } 3543 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3544 .addDef(NewReg) 3545 .addUse(GR.getSPIRVTypeID(ResType)) 3546 .constrainAllUses(TII, TRI, RBI); 3547 } 3548 assert(NewReg != ResVReg); 3549 return BuildCOPY(ResVReg, NewReg, I); 3550 } 3551 auto GlobalVar = cast<GlobalVariable>(GV); 3552 assert(GlobalVar->getName() != "llvm.global.annotations"); 3553 3554 // Skip empty declaration for GVs with initializers till we get the decl with 3555 // passed initializer. 3556 if (hasInitializer(GlobalVar) && !Init) 3557 return true; 3558 3559 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage(); 3560 SPIRV::LinkageType::LinkageType LnkType = 3561 GV->isDeclarationForLinker() 3562 ? SPIRV::LinkageType::Import 3563 : (GV->hasLinkOnceODRLinkage() && 3564 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3565 ? SPIRV::LinkageType::LinkOnceODR 3566 : SPIRV::LinkageType::Export); 3567 3568 const unsigned AddrSpace = GV->getAddressSpace(); 3569 SPIRV::StorageClass::StorageClass StorageClass = 3570 addressSpaceToStorageClass(AddrSpace, STI); 3571 SPIRVType *ResType = 3572 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3573 Register Reg = GR.buildGlobalVariable( 3574 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3575 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3576 return Reg.isValid(); 3577 } 3578 3579 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3580 const SPIRVType *ResType, 3581 MachineInstr &I) const { 3582 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3583 return selectExtInst(ResVReg, ResType, I, CL::log10); 3584 } 3585 3586 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3587 // is implemented as: 3588 // log10(x) = log2(x) * (1 / log2(10)) 3589 // = log2(x) * 0.30103 3590 3591 MachineIRBuilder MIRBuilder(I); 3592 MachineBasicBlock &BB = *I.getParent(); 3593 3594 // Build log2(x). 3595 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3596 bool Result = 3597 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3598 .addDef(VarReg) 3599 .addUse(GR.getSPIRVTypeID(ResType)) 3600 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3601 .addImm(GL::Log2) 3602 .add(I.getOperand(1)) 3603 .constrainAllUses(TII, TRI, RBI); 3604 3605 // Build 0.30103. 3606 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3607 ResType->getOpcode() == SPIRV::OpTypeFloat); 3608 // TODO: Add matrix implementation once supported by the HLSL frontend. 3609 const SPIRVType *SpirvScalarType = 3610 ResType->getOpcode() == SPIRV::OpTypeVector 3611 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3612 : ResType; 3613 Register ScaleReg = 3614 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3615 3616 // Multiply log2(x) by 0.30103 to get log10(x) result. 3617 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3618 ? SPIRV::OpVectorTimesScalar 3619 : SPIRV::OpFMulS; 3620 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3621 .addDef(ResVReg) 3622 .addUse(GR.getSPIRVTypeID(ResType)) 3623 .addUse(VarReg) 3624 .addUse(ScaleReg) 3625 .constrainAllUses(TII, TRI, RBI); 3626 } 3627 3628 // Generate the instructions to load 3-element vector builtin input 3629 // IDs/Indices. 3630 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3631 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3632 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3633 const SPIRVType *ResType, MachineInstr &I) const { 3634 MachineIRBuilder MIRBuilder(I); 3635 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3636 const SPIRVType *Vec3Ty = 3637 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3638 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3639 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3640 3641 // Create new register for the input ID builtin variable. 3642 Register NewRegister = 3643 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3644 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3645 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3646 3647 // Build global variable with the necessary decorations for the input ID 3648 // builtin variable. 3649 Register Variable = GR.buildGlobalVariable( 3650 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3651 SPIRV::StorageClass::Input, nullptr, true, true, 3652 SPIRV::LinkageType::Import, MIRBuilder, false); 3653 3654 // Create new register for loading value. 3655 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3656 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3657 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3658 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3659 3660 // Load v3uint value from the global variable. 3661 bool Result = 3662 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3663 .addDef(LoadedRegister) 3664 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3665 .addUse(Variable); 3666 3667 // Get the input ID index. Expecting operand is a constant immediate value, 3668 // wrapped in a type assignment. 3669 assert(I.getOperand(2).isReg()); 3670 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3671 3672 // Extract the input ID from the loaded vector value. 3673 MachineBasicBlock &BB = *I.getParent(); 3674 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3675 .addDef(ResVReg) 3676 .addUse(GR.getSPIRVTypeID(ResType)) 3677 .addUse(LoadedRegister) 3678 .addImm(ThreadId); 3679 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3680 } 3681 3682 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3683 MachineInstr &I) const { 3684 MachineIRBuilder MIRBuilder(I); 3685 if (Type->getOpcode() != SPIRV::OpTypeVector) 3686 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3687 3688 uint64_t VectorSize = Type->getOperand(2).getImm(); 3689 if (VectorSize == 4) 3690 return Type; 3691 3692 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3693 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3694 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3695 } 3696 3697 namespace llvm { 3698 InstructionSelector * 3699 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3700 const SPIRVSubtarget &Subtarget, 3701 const RegisterBankInfo &RBI) { 3702 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3703 } 3704 } // namespace llvm 3705