1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 #include "llvm/Support/ErrorHandling.h" 37 38 #define DEBUG_TYPE "spirv-isel" 39 40 using namespace llvm; 41 namespace CL = SPIRV::OpenCLExtInst; 42 namespace GL = SPIRV::GLSLExtInst; 43 44 using ExtInstList = 45 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 46 47 namespace { 48 49 llvm::SPIRV::SelectionControl::SelectionControl 50 getSelectionOperandForImm(int Imm) { 51 if (Imm == 2) 52 return SPIRV::SelectionControl::Flatten; 53 if (Imm == 1) 54 return SPIRV::SelectionControl::DontFlatten; 55 if (Imm == 0) 56 return SPIRV::SelectionControl::None; 57 llvm_unreachable("Invalid immediate"); 58 } 59 60 #define GET_GLOBALISEL_PREDICATE_BITSET 61 #include "SPIRVGenGlobalISel.inc" 62 #undef GET_GLOBALISEL_PREDICATE_BITSET 63 64 class SPIRVInstructionSelector : public InstructionSelector { 65 const SPIRVSubtarget &STI; 66 const SPIRVInstrInfo &TII; 67 const SPIRVRegisterInfo &TRI; 68 const RegisterBankInfo &RBI; 69 SPIRVGlobalRegistry &GR; 70 MachineRegisterInfo *MRI; 71 MachineFunction *HasVRegsReset = nullptr; 72 73 /// We need to keep track of the number we give to anonymous global values to 74 /// generate the same name every time when this is needed. 75 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 76 SmallPtrSet<MachineInstr *, 8> DeadMIs; 77 78 public: 79 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 80 const SPIRVSubtarget &ST, 81 const RegisterBankInfo &RBI); 82 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 83 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 84 BlockFrequencyInfo *BFI) override; 85 // Common selection code. Instruction-specific selection occurs in spvSelect. 86 bool select(MachineInstr &I) override; 87 static const char *getName() { return DEBUG_TYPE; } 88 89 #define GET_GLOBALISEL_PREDICATES_DECL 90 #include "SPIRVGenGlobalISel.inc" 91 #undef GET_GLOBALISEL_PREDICATES_DECL 92 93 #define GET_GLOBALISEL_TEMPORARIES_DECL 94 #include "SPIRVGenGlobalISel.inc" 95 #undef GET_GLOBALISEL_TEMPORARIES_DECL 96 97 private: 98 void resetVRegsType(MachineFunction &MF); 99 100 // tblgen-erated 'select' implementation, used as the initial selector for 101 // the patterns that don't require complex C++. 102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 103 104 // All instruction-specific selection that didn't happen in "select()". 105 // Is basically a large Switch/Case delegating to all other select method. 106 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I) const; 108 109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 110 MachineInstr &I, bool IsSigned) const; 111 112 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I) const; 114 115 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, unsigned ExtendOpcode, 117 unsigned BitSetOpcode) const; 118 119 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I, Register SrcReg, 121 unsigned BitSetOpcode) const; 122 123 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I, Register SrcReg, 125 unsigned BitSetOpcode, bool SwapPrimarySide) const; 126 127 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I, Register SrcReg, 129 unsigned BitSetOpcode, 130 bool SwapPrimarySide) const; 131 132 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 133 const MachineInstr *Init = nullptr) const; 134 135 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I, std::vector<Register> SrcRegs, 137 unsigned Opcode) const; 138 139 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 140 unsigned Opcode) const; 141 142 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 143 MachineInstr &I) const; 144 145 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 bool selectStore(MachineInstr &I) const; 148 149 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 bool selectStackRestore(MachineInstr &I) const; 152 153 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 154 155 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I, unsigned NewOpcode, 157 unsigned NegateOpcode = 0) const; 158 159 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 162 bool selectFence(MachineInstr &I) const; 163 164 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I, unsigned OpType) const; 169 170 bool selectAll(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectAny(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I) const; 181 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 185 unsigned comparisonOpcode, MachineInstr &I) const; 186 bool selectCross(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 192 MachineInstr &I) const; 193 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 194 MachineInstr &I) const; 195 196 bool selectSign(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 199 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 200 MachineInstr &I) const; 201 202 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I, unsigned Opcode) const; 204 205 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I, bool Signed) const; 207 208 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 209 MachineInstr &I) const; 210 211 template <bool Signed> 212 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 template <bool Signed> 215 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 216 MachineInstr &I) const; 217 218 bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I) const; 220 221 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 222 int OpIdx) const; 223 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 224 int OpIdx) const; 225 226 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 227 MachineInstr &I) const; 228 229 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 230 bool IsSigned) const; 231 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 232 bool IsSigned, unsigned Opcode) const; 233 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 234 bool IsSigned) const; 235 236 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 237 MachineInstr &I) const; 238 239 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 240 bool IsSigned) const; 241 242 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 243 const SPIRVType *intTy, const SPIRVType *boolTy) const; 244 245 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 246 MachineInstr &I) const; 247 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 248 MachineInstr &I) const; 249 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 250 MachineInstr &I) const; 251 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I) const; 253 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 254 MachineInstr &I) const; 255 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 256 MachineInstr &I) const; 257 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I) const; 259 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 262 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 263 MachineInstr &I) const; 264 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 265 MachineInstr &I) const; 266 267 bool selectBranch(MachineInstr &I) const; 268 bool selectBranchCond(MachineInstr &I) const; 269 270 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 271 MachineInstr &I) const; 272 273 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 274 MachineInstr &I, GL::GLSLExtInst GLInst) const; 275 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 276 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 277 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 278 MachineInstr &I, CL::OpenCLExtInst CLInst, 279 GL::GLSLExtInst GLInst) const; 280 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 281 MachineInstr &I, const ExtInstList &ExtInsts) const; 282 283 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 284 MachineInstr &I) const; 285 286 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 287 MachineInstr &I) const; 288 289 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 290 MachineInstr &I, unsigned Opcode) const; 291 292 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 293 MachineInstr &I) const; 294 295 bool selectUnmergeValues(MachineInstr &I) const; 296 297 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 298 MachineInstr &I) const; 299 300 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 301 MachineInstr &I) const; 302 bool selectImageWriteIntrinsic(MachineInstr &I) const; 303 bool selectResourceGetPointer(Register &ResVReg, const SPIRVType *ResType, 304 MachineInstr &I) const; 305 306 // Utilities 307 std::pair<Register, bool> 308 buildI32Constant(uint32_t Val, MachineInstr &I, 309 const SPIRVType *ResType = nullptr) const; 310 311 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 312 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 313 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 314 MachineInstr &I) const; 315 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 316 317 bool wrapIntoSpecConstantOp(MachineInstr &I, 318 SmallVector<Register> &CompositeArgs) const; 319 320 Register getUcharPtrTypeReg(MachineInstr &I, 321 SPIRV::StorageClass::StorageClass SC) const; 322 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 323 Register Src, Register DestType, 324 uint32_t Opcode) const; 325 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 326 SPIRVType *SrcPtrTy) const; 327 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 328 uint32_t Binding, uint32_t ArraySize, 329 Register IndexReg, bool IsNonUniform, 330 MachineIRBuilder MIRBuilder) const; 331 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 332 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType, 333 Register &ReadReg, MachineInstr &InsertionPoint) const; 334 bool generateImageRead(Register &ResVReg, const SPIRVType *ResType, 335 Register ImageReg, Register IdxReg, DebugLoc Loc, 336 MachineInstr &Pos) const; 337 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 338 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 339 Register ResVReg, const SPIRVType *ResType, 340 MachineInstr &I) const; 341 bool loadHandleBeforePosition(Register &HandleReg, const SPIRVType *ResType, 342 GIntrinsic &HandleDef, MachineInstr &Pos) const; 343 }; 344 345 } // end anonymous namespace 346 347 #define GET_GLOBALISEL_IMPL 348 #include "SPIRVGenGlobalISel.inc" 349 #undef GET_GLOBALISEL_IMPL 350 351 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 352 const SPIRVSubtarget &ST, 353 const RegisterBankInfo &RBI) 354 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 355 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 356 #define GET_GLOBALISEL_PREDICATES_INIT 357 #include "SPIRVGenGlobalISel.inc" 358 #undef GET_GLOBALISEL_PREDICATES_INIT 359 #define GET_GLOBALISEL_TEMPORARIES_INIT 360 #include "SPIRVGenGlobalISel.inc" 361 #undef GET_GLOBALISEL_TEMPORARIES_INIT 362 { 363 } 364 365 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 366 CodeGenCoverage *CoverageInfo, 367 ProfileSummaryInfo *PSI, 368 BlockFrequencyInfo *BFI) { 369 MRI = &MF.getRegInfo(); 370 GR.setCurrentFunc(MF); 371 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 372 } 373 374 // Ensure that register classes correspond to pattern matching rules. 375 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 376 if (HasVRegsReset == &MF) 377 return; 378 HasVRegsReset = &MF; 379 380 MachineRegisterInfo &MRI = MF.getRegInfo(); 381 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 382 Register Reg = Register::index2VirtReg(I); 383 LLT RegType = MRI.getType(Reg); 384 if (RegType.isScalar()) 385 MRI.setType(Reg, LLT::scalar(64)); 386 else if (RegType.isPointer()) 387 MRI.setType(Reg, LLT::pointer(0, 64)); 388 else if (RegType.isVector()) 389 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 390 } 391 for (const auto &MBB : MF) { 392 for (const auto &MI : MBB) { 393 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 394 continue; 395 Register DstReg = MI.getOperand(0).getReg(); 396 LLT DstType = MRI.getType(DstReg); 397 Register SrcReg = MI.getOperand(1).getReg(); 398 LLT SrcType = MRI.getType(SrcReg); 399 if (DstType != SrcType) 400 MRI.setType(DstReg, MRI.getType(SrcReg)); 401 402 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 403 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 404 if (DstRC != SrcRC && SrcRC) 405 MRI.setRegClass(DstReg, SrcRC); 406 } 407 } 408 } 409 410 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 411 412 // Defined in SPIRVLegalizerInfo.cpp. 413 extern bool isTypeFoldingSupported(unsigned Opcode); 414 415 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 416 for (const auto &MO : MI.all_defs()) { 417 Register Reg = MO.getReg(); 418 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 419 return false; 420 } 421 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 422 MI.isLifetimeMarker()) 423 return false; 424 if (MI.isPHI()) 425 return true; 426 if (MI.mayStore() || MI.isCall() || 427 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 428 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 429 return false; 430 return true; 431 } 432 433 bool SPIRVInstructionSelector::select(MachineInstr &I) { 434 resetVRegsType(*I.getParent()->getParent()); 435 436 assert(I.getParent() && "Instruction should be in a basic block!"); 437 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 438 439 Register Opcode = I.getOpcode(); 440 // If it's not a GMIR instruction, we've selected it already. 441 if (!isPreISelGenericOpcode(Opcode)) { 442 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 443 Register DstReg = I.getOperand(0).getReg(); 444 Register SrcReg = I.getOperand(1).getReg(); 445 auto *Def = MRI->getVRegDef(SrcReg); 446 if (isTypeFoldingSupported(Def->getOpcode())) { 447 bool Res = selectImpl(I, *CoverageInfo); 448 LLVM_DEBUG({ 449 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 450 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 451 I.print(dbgs()); 452 } 453 }); 454 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 455 if (Res) { 456 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 457 DeadMIs.insert(Def); 458 return Res; 459 } 460 } 461 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 462 MRI->replaceRegWith(SrcReg, DstReg); 463 GR.invalidateMachineInstr(&I); 464 I.removeFromParent(); 465 return true; 466 } else if (I.getNumDefs() == 1) { 467 // Make all vregs 64 bits (for SPIR-V IDs). 468 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 469 } 470 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 471 } 472 473 if (DeadMIs.contains(&I)) { 474 // if the instruction has been already made dead by folding it away 475 // erase it 476 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 477 salvageDebugInfo(*MRI, I); 478 GR.invalidateMachineInstr(&I); 479 I.eraseFromParent(); 480 return true; 481 } 482 483 if (I.getNumOperands() != I.getNumExplicitOperands()) { 484 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 485 return false; 486 } 487 488 // Common code for getting return reg+type, and removing selected instr 489 // from parent occurs here. Instr-specific selection happens in spvSelect(). 490 bool HasDefs = I.getNumDefs() > 0; 491 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 492 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 493 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 494 if (spvSelect(ResVReg, ResType, I)) { 495 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 496 for (unsigned i = 0; i < I.getNumDefs(); ++i) 497 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 498 GR.invalidateMachineInstr(&I); 499 I.removeFromParent(); 500 return true; 501 } 502 return false; 503 } 504 505 static bool mayApplyGenericSelection(unsigned Opcode) { 506 switch (Opcode) { 507 case TargetOpcode::G_CONSTANT: 508 return false; 509 case TargetOpcode::G_SADDO: 510 case TargetOpcode::G_SSUBO: 511 return true; 512 } 513 return isTypeFoldingSupported(Opcode); 514 } 515 516 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 517 MachineInstr &I) const { 518 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 519 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 520 if (DstRC != SrcRC && SrcRC) 521 MRI->setRegClass(DestReg, SrcRC); 522 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 523 TII.get(TargetOpcode::COPY)) 524 .addDef(DestReg) 525 .addUse(SrcReg) 526 .constrainAllUses(TII, TRI, RBI); 527 } 528 529 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 530 const SPIRVType *ResType, 531 MachineInstr &I) const { 532 const unsigned Opcode = I.getOpcode(); 533 if (mayApplyGenericSelection(Opcode)) 534 return selectImpl(I, *CoverageInfo); 535 switch (Opcode) { 536 case TargetOpcode::G_CONSTANT: 537 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 538 I); 539 case TargetOpcode::G_GLOBAL_VALUE: 540 return selectGlobalValue(ResVReg, I); 541 case TargetOpcode::G_IMPLICIT_DEF: 542 return selectOpUndef(ResVReg, ResType, I); 543 case TargetOpcode::G_FREEZE: 544 return selectFreeze(ResVReg, ResType, I); 545 546 case TargetOpcode::G_INTRINSIC: 547 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 548 case TargetOpcode::G_INTRINSIC_CONVERGENT: 549 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 550 return selectIntrinsic(ResVReg, ResType, I); 551 case TargetOpcode::G_BITREVERSE: 552 return selectBitreverse(ResVReg, ResType, I); 553 554 case TargetOpcode::G_BUILD_VECTOR: 555 return selectBuildVector(ResVReg, ResType, I); 556 case TargetOpcode::G_SPLAT_VECTOR: 557 return selectSplatVector(ResVReg, ResType, I); 558 559 case TargetOpcode::G_SHUFFLE_VECTOR: { 560 MachineBasicBlock &BB = *I.getParent(); 561 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 562 .addDef(ResVReg) 563 .addUse(GR.getSPIRVTypeID(ResType)) 564 .addUse(I.getOperand(1).getReg()) 565 .addUse(I.getOperand(2).getReg()); 566 for (auto V : I.getOperand(3).getShuffleMask()) 567 MIB.addImm(V); 568 return MIB.constrainAllUses(TII, TRI, RBI); 569 } 570 case TargetOpcode::G_MEMMOVE: 571 case TargetOpcode::G_MEMCPY: 572 case TargetOpcode::G_MEMSET: 573 return selectMemOperation(ResVReg, I); 574 575 case TargetOpcode::G_ICMP: 576 return selectICmp(ResVReg, ResType, I); 577 case TargetOpcode::G_FCMP: 578 return selectFCmp(ResVReg, ResType, I); 579 580 case TargetOpcode::G_FRAME_INDEX: 581 return selectFrameIndex(ResVReg, ResType, I); 582 583 case TargetOpcode::G_LOAD: 584 return selectLoad(ResVReg, ResType, I); 585 case TargetOpcode::G_STORE: 586 return selectStore(I); 587 588 case TargetOpcode::G_BR: 589 return selectBranch(I); 590 case TargetOpcode::G_BRCOND: 591 return selectBranchCond(I); 592 593 case TargetOpcode::G_PHI: 594 return selectPhi(ResVReg, ResType, I); 595 596 case TargetOpcode::G_FPTOSI: 597 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 598 case TargetOpcode::G_FPTOUI: 599 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 600 601 case TargetOpcode::G_SITOFP: 602 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 603 case TargetOpcode::G_UITOFP: 604 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 605 606 case TargetOpcode::G_CTPOP: 607 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 608 case TargetOpcode::G_SMIN: 609 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 610 case TargetOpcode::G_UMIN: 611 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 612 613 case TargetOpcode::G_SMAX: 614 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 615 case TargetOpcode::G_UMAX: 616 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 617 618 case TargetOpcode::G_SCMP: 619 return selectSUCmp(ResVReg, ResType, I, true); 620 case TargetOpcode::G_UCMP: 621 return selectSUCmp(ResVReg, ResType, I, false); 622 623 case TargetOpcode::G_STRICT_FMA: 624 case TargetOpcode::G_FMA: 625 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 626 627 case TargetOpcode::G_STRICT_FLDEXP: 628 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 629 630 case TargetOpcode::G_FPOW: 631 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 632 case TargetOpcode::G_FPOWI: 633 return selectExtInst(ResVReg, ResType, I, CL::pown); 634 635 case TargetOpcode::G_FEXP: 636 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 637 case TargetOpcode::G_FEXP2: 638 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 639 640 case TargetOpcode::G_FLOG: 641 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 642 case TargetOpcode::G_FLOG2: 643 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 644 case TargetOpcode::G_FLOG10: 645 return selectLog10(ResVReg, ResType, I); 646 647 case TargetOpcode::G_FABS: 648 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 649 case TargetOpcode::G_ABS: 650 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 651 652 case TargetOpcode::G_FMINNUM: 653 case TargetOpcode::G_FMINIMUM: 654 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 655 case TargetOpcode::G_FMAXNUM: 656 case TargetOpcode::G_FMAXIMUM: 657 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 658 659 case TargetOpcode::G_FCOPYSIGN: 660 return selectExtInst(ResVReg, ResType, I, CL::copysign); 661 662 case TargetOpcode::G_FCEIL: 663 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 664 case TargetOpcode::G_FFLOOR: 665 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 666 667 case TargetOpcode::G_FCOS: 668 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 669 case TargetOpcode::G_FSIN: 670 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 671 case TargetOpcode::G_FTAN: 672 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 673 case TargetOpcode::G_FACOS: 674 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 675 case TargetOpcode::G_FASIN: 676 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 677 case TargetOpcode::G_FATAN: 678 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 679 case TargetOpcode::G_FATAN2: 680 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 681 case TargetOpcode::G_FCOSH: 682 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 683 case TargetOpcode::G_FSINH: 684 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 685 case TargetOpcode::G_FTANH: 686 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 687 688 case TargetOpcode::G_STRICT_FSQRT: 689 case TargetOpcode::G_FSQRT: 690 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 691 692 case TargetOpcode::G_CTTZ: 693 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 694 return selectExtInst(ResVReg, ResType, I, CL::ctz); 695 case TargetOpcode::G_CTLZ: 696 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 697 return selectExtInst(ResVReg, ResType, I, CL::clz); 698 699 case TargetOpcode::G_INTRINSIC_ROUND: 700 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 701 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 702 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 703 case TargetOpcode::G_INTRINSIC_TRUNC: 704 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 705 case TargetOpcode::G_FRINT: 706 case TargetOpcode::G_FNEARBYINT: 707 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 708 709 case TargetOpcode::G_SMULH: 710 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 711 case TargetOpcode::G_UMULH: 712 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 713 714 case TargetOpcode::G_SADDSAT: 715 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 716 case TargetOpcode::G_UADDSAT: 717 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 718 case TargetOpcode::G_SSUBSAT: 719 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 720 case TargetOpcode::G_USUBSAT: 721 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 722 723 case TargetOpcode::G_UADDO: 724 return selectOverflowArith(ResVReg, ResType, I, 725 ResType->getOpcode() == SPIRV::OpTypeVector 726 ? SPIRV::OpIAddCarryV 727 : SPIRV::OpIAddCarryS); 728 case TargetOpcode::G_USUBO: 729 return selectOverflowArith(ResVReg, ResType, I, 730 ResType->getOpcode() == SPIRV::OpTypeVector 731 ? SPIRV::OpISubBorrowV 732 : SPIRV::OpISubBorrowS); 733 case TargetOpcode::G_UMULO: 734 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 735 case TargetOpcode::G_SMULO: 736 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 737 738 case TargetOpcode::G_SEXT: 739 return selectExt(ResVReg, ResType, I, true); 740 case TargetOpcode::G_ANYEXT: 741 case TargetOpcode::G_ZEXT: 742 return selectExt(ResVReg, ResType, I, false); 743 case TargetOpcode::G_TRUNC: 744 return selectTrunc(ResVReg, ResType, I); 745 case TargetOpcode::G_FPTRUNC: 746 case TargetOpcode::G_FPEXT: 747 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 748 749 case TargetOpcode::G_PTRTOINT: 750 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 751 case TargetOpcode::G_INTTOPTR: 752 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 753 case TargetOpcode::G_BITCAST: 754 return selectBitcast(ResVReg, ResType, I); 755 case TargetOpcode::G_ADDRSPACE_CAST: 756 return selectAddrSpaceCast(ResVReg, ResType, I); 757 case TargetOpcode::G_PTR_ADD: { 758 // Currently, we get G_PTR_ADD only applied to global variables. 759 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 760 Register GV = I.getOperand(1).getReg(); 761 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 762 (void)II; 763 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 764 (*II).getOpcode() == TargetOpcode::COPY || 765 (*II).getOpcode() == SPIRV::OpVariable) && 766 isImm(I.getOperand(2), MRI)); 767 // It may be the initialization of a global variable. 768 bool IsGVInit = false; 769 for (MachineRegisterInfo::use_instr_iterator 770 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 771 UseEnd = MRI->use_instr_end(); 772 UseIt != UseEnd; UseIt = std::next(UseIt)) { 773 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 774 (*UseIt).getOpcode() == SPIRV::OpVariable) { 775 IsGVInit = true; 776 break; 777 } 778 } 779 MachineBasicBlock &BB = *I.getParent(); 780 if (!IsGVInit) { 781 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 782 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 783 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 784 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 785 // Build a new virtual register that is associated with the required 786 // data type. 787 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 788 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 789 // Having a correctly typed base we are ready to build the actually 790 // required GEP. It may not be a constant though, because all Operands 791 // of OpSpecConstantOp is to originate from other const instructions, 792 // and only the AccessChain named opcodes accept a global OpVariable 793 // instruction. We can't use an AccessChain opcode because of the type 794 // mismatch between result and base types. 795 if (!GR.isBitcastCompatible(ResType, GVType)) 796 report_fatal_error( 797 "incompatible result and operand types in a bitcast"); 798 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 799 MachineInstrBuilder MIB = 800 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 801 .addDef(NewVReg) 802 .addUse(ResTypeReg) 803 .addUse(GV); 804 return MIB.constrainAllUses(TII, TRI, RBI) && 805 BuildMI(BB, I, I.getDebugLoc(), 806 TII.get(STI.isVulkanEnv() 807 ? SPIRV::OpInBoundsAccessChain 808 : SPIRV::OpInBoundsPtrAccessChain)) 809 .addDef(ResVReg) 810 .addUse(ResTypeReg) 811 .addUse(NewVReg) 812 .addUse(I.getOperand(2).getReg()) 813 .constrainAllUses(TII, TRI, RBI); 814 } else { 815 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 816 .addDef(ResVReg) 817 .addUse(GR.getSPIRVTypeID(ResType)) 818 .addImm( 819 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 820 .addUse(GV) 821 .addUse(I.getOperand(2).getReg()) 822 .constrainAllUses(TII, TRI, RBI); 823 } 824 } 825 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 826 // initialize a global variable with a constant expression (e.g., the test 827 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 828 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 829 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 830 .addDef(ResVReg) 831 .addUse(GR.getSPIRVTypeID(ResType)) 832 .addImm(static_cast<uint32_t>( 833 SPIRV::Opcode::InBoundsPtrAccessChain)) 834 .addUse(GV) 835 .addUse(Idx) 836 .addUse(I.getOperand(2).getReg()); 837 return MIB.constrainAllUses(TII, TRI, RBI); 838 } 839 840 case TargetOpcode::G_ATOMICRMW_OR: 841 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 842 case TargetOpcode::G_ATOMICRMW_ADD: 843 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 844 case TargetOpcode::G_ATOMICRMW_AND: 845 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 846 case TargetOpcode::G_ATOMICRMW_MAX: 847 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 848 case TargetOpcode::G_ATOMICRMW_MIN: 849 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 850 case TargetOpcode::G_ATOMICRMW_SUB: 851 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 852 case TargetOpcode::G_ATOMICRMW_XOR: 853 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 854 case TargetOpcode::G_ATOMICRMW_UMAX: 855 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 856 case TargetOpcode::G_ATOMICRMW_UMIN: 857 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 858 case TargetOpcode::G_ATOMICRMW_XCHG: 859 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 860 case TargetOpcode::G_ATOMIC_CMPXCHG: 861 return selectAtomicCmpXchg(ResVReg, ResType, I); 862 863 case TargetOpcode::G_ATOMICRMW_FADD: 864 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 865 case TargetOpcode::G_ATOMICRMW_FSUB: 866 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 867 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 868 SPIRV::OpFNegate); 869 case TargetOpcode::G_ATOMICRMW_FMIN: 870 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 871 case TargetOpcode::G_ATOMICRMW_FMAX: 872 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 873 874 case TargetOpcode::G_FENCE: 875 return selectFence(I); 876 877 case TargetOpcode::G_STACKSAVE: 878 return selectStackSave(ResVReg, ResType, I); 879 case TargetOpcode::G_STACKRESTORE: 880 return selectStackRestore(I); 881 882 case TargetOpcode::G_UNMERGE_VALUES: 883 return selectUnmergeValues(I); 884 885 // Discard gen opcodes for intrinsics which we do not expect to actually 886 // represent code after lowering or intrinsics which are not implemented but 887 // should not crash when found in a customer's LLVM IR input. 888 case TargetOpcode::G_TRAP: 889 case TargetOpcode::G_DEBUGTRAP: 890 case TargetOpcode::G_UBSANTRAP: 891 case TargetOpcode::DBG_LABEL: 892 return true; 893 894 default: 895 return false; 896 } 897 } 898 899 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 900 const SPIRVType *ResType, 901 MachineInstr &I, 902 GL::GLSLExtInst GLInst) const { 903 if (!STI.canUseExtInstSet( 904 SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) { 905 std::string DiagMsg; 906 raw_string_ostream OS(DiagMsg); 907 I.print(OS, true, false, false, false); 908 DiagMsg += " is only supported with the GLSL extended instruction set.\n"; 909 report_fatal_error(DiagMsg.c_str(), false); 910 } 911 return selectExtInst(ResVReg, ResType, I, 912 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 913 } 914 915 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 916 const SPIRVType *ResType, 917 MachineInstr &I, 918 CL::OpenCLExtInst CLInst) const { 919 return selectExtInst(ResVReg, ResType, I, 920 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 921 } 922 923 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 924 const SPIRVType *ResType, 925 MachineInstr &I, 926 CL::OpenCLExtInst CLInst, 927 GL::GLSLExtInst GLInst) const { 928 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 929 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 930 return selectExtInst(ResVReg, ResType, I, ExtInsts); 931 } 932 933 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 934 const SPIRVType *ResType, 935 MachineInstr &I, 936 const ExtInstList &Insts) const { 937 938 for (const auto &Ex : Insts) { 939 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 940 uint32_t Opcode = Ex.second; 941 if (STI.canUseExtInstSet(Set)) { 942 MachineBasicBlock &BB = *I.getParent(); 943 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 944 .addDef(ResVReg) 945 .addUse(GR.getSPIRVTypeID(ResType)) 946 .addImm(static_cast<uint32_t>(Set)) 947 .addImm(Opcode); 948 const unsigned NumOps = I.getNumOperands(); 949 unsigned Index = 1; 950 if (Index < NumOps && 951 I.getOperand(Index).getType() == 952 MachineOperand::MachineOperandType::MO_IntrinsicID) 953 Index = 2; 954 for (; Index < NumOps; ++Index) 955 MIB.add(I.getOperand(Index)); 956 return MIB.constrainAllUses(TII, TRI, RBI); 957 } 958 } 959 return false; 960 } 961 962 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 963 const SPIRVType *ResType, 964 MachineInstr &I, 965 std::vector<Register> Srcs, 966 unsigned Opcode) const { 967 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 968 .addDef(ResVReg) 969 .addUse(GR.getSPIRVTypeID(ResType)); 970 for (Register SReg : Srcs) { 971 MIB.addUse(SReg); 972 } 973 return MIB.constrainAllUses(TII, TRI, RBI); 974 } 975 976 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 977 const SPIRVType *ResType, 978 MachineInstr &I, 979 unsigned Opcode) const { 980 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 981 Register SrcReg = I.getOperand(1).getReg(); 982 bool IsGV = false; 983 for (MachineRegisterInfo::def_instr_iterator DefIt = 984 MRI->def_instr_begin(SrcReg); 985 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 986 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 987 IsGV = true; 988 break; 989 } 990 } 991 if (IsGV) { 992 uint32_t SpecOpcode = 0; 993 switch (Opcode) { 994 case SPIRV::OpConvertPtrToU: 995 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 996 break; 997 case SPIRV::OpConvertUToPtr: 998 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 999 break; 1000 } 1001 if (SpecOpcode) 1002 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1003 TII.get(SPIRV::OpSpecConstantOp)) 1004 .addDef(ResVReg) 1005 .addUse(GR.getSPIRVTypeID(ResType)) 1006 .addImm(SpecOpcode) 1007 .addUse(SrcReg) 1008 .constrainAllUses(TII, TRI, RBI); 1009 } 1010 } 1011 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 1012 Opcode); 1013 } 1014 1015 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 1016 const SPIRVType *ResType, 1017 MachineInstr &I) const { 1018 Register OpReg = I.getOperand(1).getReg(); 1019 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 1020 if (!GR.isBitcastCompatible(ResType, OpType)) 1021 report_fatal_error("incompatible result and operand types in a bitcast"); 1022 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1023 } 1024 1025 static void addMemoryOperands(MachineMemOperand *MemOp, 1026 MachineInstrBuilder &MIB) { 1027 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1028 if (MemOp->isVolatile()) 1029 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1030 if (MemOp->isNonTemporal()) 1031 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1032 if (MemOp->getAlign().value()) 1033 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 1034 1035 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 1036 MIB.addImm(SpvMemOp); 1037 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 1038 MIB.addImm(MemOp->getAlign().value()); 1039 } 1040 } 1041 1042 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1043 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1044 if (Flags & MachineMemOperand::Flags::MOVolatile) 1045 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1046 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1047 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1048 1049 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1050 MIB.addImm(SpvMemOp); 1051 } 1052 1053 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1054 const SPIRVType *ResType, 1055 MachineInstr &I) const { 1056 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1057 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1058 1059 auto *PtrDef = getVRegDef(*MRI, Ptr); 1060 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); 1061 if (IntPtrDef && 1062 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 1063 Register ImageReg = IntPtrDef->getOperand(2).getReg(); 1064 Register NewImageReg = 1065 MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 1066 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 1067 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 1068 *ImageDef, I)) { 1069 return false; 1070 } 1071 1072 Register IdxReg = IntPtrDef->getOperand(3).getReg(); 1073 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, 1074 I.getDebugLoc(), I); 1075 } 1076 1077 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1078 .addDef(ResVReg) 1079 .addUse(GR.getSPIRVTypeID(ResType)) 1080 .addUse(Ptr); 1081 if (!I.getNumMemOperands()) { 1082 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1083 I.getOpcode() == 1084 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1085 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1086 } else { 1087 addMemoryOperands(*I.memoperands_begin(), MIB); 1088 } 1089 return MIB.constrainAllUses(TII, TRI, RBI); 1090 } 1091 1092 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1093 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1094 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1095 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1096 1097 auto *PtrDef = getVRegDef(*MRI, Ptr); 1098 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); 1099 if (IntPtrDef && 1100 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 1101 Register ImageReg = IntPtrDef->getOperand(2).getReg(); 1102 Register NewImageReg = 1103 MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 1104 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 1105 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 1106 *ImageDef, I)) { 1107 return false; 1108 } 1109 1110 Register IdxReg = IntPtrDef->getOperand(3).getReg(); 1111 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1112 TII.get(SPIRV::OpImageWrite)) 1113 .addUse(NewImageReg) 1114 .addUse(IdxReg) 1115 .addUse(StoreVal) 1116 .constrainAllUses(TII, TRI, RBI); 1117 } 1118 1119 MachineBasicBlock &BB = *I.getParent(); 1120 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1121 .addUse(Ptr) 1122 .addUse(StoreVal); 1123 if (!I.getNumMemOperands()) { 1124 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1125 I.getOpcode() == 1126 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1127 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1128 } else { 1129 addMemoryOperands(*I.memoperands_begin(), MIB); 1130 } 1131 return MIB.constrainAllUses(TII, TRI, RBI); 1132 } 1133 1134 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1135 const SPIRVType *ResType, 1136 MachineInstr &I) const { 1137 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1138 report_fatal_error( 1139 "llvm.stacksave intrinsic: this instruction requires the following " 1140 "SPIR-V extension: SPV_INTEL_variable_length_array", 1141 false); 1142 MachineBasicBlock &BB = *I.getParent(); 1143 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1144 .addDef(ResVReg) 1145 .addUse(GR.getSPIRVTypeID(ResType)) 1146 .constrainAllUses(TII, TRI, RBI); 1147 } 1148 1149 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1150 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1151 report_fatal_error( 1152 "llvm.stackrestore intrinsic: this instruction requires the following " 1153 "SPIR-V extension: SPV_INTEL_variable_length_array", 1154 false); 1155 if (!I.getOperand(0).isReg()) 1156 return false; 1157 MachineBasicBlock &BB = *I.getParent(); 1158 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1159 .addUse(I.getOperand(0).getReg()) 1160 .constrainAllUses(TII, TRI, RBI); 1161 } 1162 1163 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1164 MachineInstr &I) const { 1165 MachineBasicBlock &BB = *I.getParent(); 1166 Register SrcReg = I.getOperand(1).getReg(); 1167 bool Result = true; 1168 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1169 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1170 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1171 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1172 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1173 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1174 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1175 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1176 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1177 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1178 Function &CurFunction = GR.CurMF->getFunction(); 1179 Type *LLVMArrTy = 1180 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1181 // Module takes ownership of the global var. 1182 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1183 true, GlobalValue::InternalLinkage, 1184 Constant::getNullValue(LLVMArrTy)); 1185 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1186 GR.add(GV, GR.CurMF, VarReg); 1187 GR.addGlobalObject(GV, GR.CurMF, VarReg); 1188 1189 Result &= 1190 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1191 .addDef(VarReg) 1192 .addUse(GR.getSPIRVTypeID(VarTy)) 1193 .addImm(SPIRV::StorageClass::UniformConstant) 1194 .addUse(Const) 1195 .constrainAllUses(TII, TRI, RBI); 1196 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1197 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1198 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1199 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1200 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1201 } 1202 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1203 .addUse(I.getOperand(0).getReg()) 1204 .addUse(SrcReg) 1205 .addUse(I.getOperand(2).getReg()); 1206 if (I.getNumMemOperands()) 1207 addMemoryOperands(*I.memoperands_begin(), MIB); 1208 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1209 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1210 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1211 return Result; 1212 } 1213 1214 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1215 const SPIRVType *ResType, 1216 MachineInstr &I, 1217 unsigned NewOpcode, 1218 unsigned NegateOpcode) const { 1219 bool Result = true; 1220 assert(I.hasOneMemOperand()); 1221 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1222 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1223 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1224 auto ScopeConstant = buildI32Constant(Scope, I); 1225 Register ScopeReg = ScopeConstant.first; 1226 Result &= ScopeConstant.second; 1227 1228 Register Ptr = I.getOperand(1).getReg(); 1229 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1230 // auto ScSem = 1231 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1232 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1233 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1234 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1235 Register MemSemReg = MemSemConstant.first; 1236 Result &= MemSemConstant.second; 1237 1238 Register ValueReg = I.getOperand(2).getReg(); 1239 if (NegateOpcode != 0) { 1240 // Translation with negative value operand is requested 1241 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1242 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1243 ValueReg = TmpReg; 1244 } 1245 1246 return Result && 1247 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1248 .addDef(ResVReg) 1249 .addUse(GR.getSPIRVTypeID(ResType)) 1250 .addUse(Ptr) 1251 .addUse(ScopeReg) 1252 .addUse(MemSemReg) 1253 .addUse(ValueReg) 1254 .constrainAllUses(TII, TRI, RBI); 1255 } 1256 1257 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1258 unsigned ArgI = I.getNumOperands() - 1; 1259 Register SrcReg = 1260 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1261 SPIRVType *DefType = 1262 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1263 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1264 report_fatal_error( 1265 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1266 1267 SPIRVType *ScalarType = 1268 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1269 MachineBasicBlock &BB = *I.getParent(); 1270 bool Res = false; 1271 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1272 Register ResVReg = I.getOperand(i).getReg(); 1273 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1274 if (!ResType) { 1275 // There was no "assign type" actions, let's fix this now 1276 ResType = ScalarType; 1277 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1278 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1279 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1280 } 1281 auto MIB = 1282 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1283 .addDef(ResVReg) 1284 .addUse(GR.getSPIRVTypeID(ResType)) 1285 .addUse(SrcReg) 1286 .addImm(static_cast<int64_t>(i)); 1287 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1288 } 1289 return Res; 1290 } 1291 1292 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1293 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1294 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1295 auto MemSemConstant = buildI32Constant(MemSem, I); 1296 Register MemSemReg = MemSemConstant.first; 1297 bool Result = MemSemConstant.second; 1298 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1299 uint32_t Scope = static_cast<uint32_t>( 1300 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1301 auto ScopeConstant = buildI32Constant(Scope, I); 1302 Register ScopeReg = ScopeConstant.first; 1303 Result &= ScopeConstant.second; 1304 MachineBasicBlock &BB = *I.getParent(); 1305 return Result && 1306 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1307 .addUse(ScopeReg) 1308 .addUse(MemSemReg) 1309 .constrainAllUses(TII, TRI, RBI); 1310 } 1311 1312 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1313 const SPIRVType *ResType, 1314 MachineInstr &I, 1315 unsigned Opcode) const { 1316 Type *ResTy = nullptr; 1317 StringRef ResName; 1318 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1319 report_fatal_error( 1320 "Not enough info to select the arithmetic with overflow instruction"); 1321 if (!ResTy || !ResTy->isStructTy()) 1322 report_fatal_error("Expect struct type result for the arithmetic " 1323 "with overflow instruction"); 1324 // "Result Type must be from OpTypeStruct. The struct must have two members, 1325 // and the two members must be the same type." 1326 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1327 ResTy = StructType::get(ResElemTy, ResElemTy); 1328 // Build SPIR-V types and constant(s) if needed. 1329 MachineIRBuilder MIRBuilder(I); 1330 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1331 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1332 assert(I.getNumDefs() > 1 && "Not enought operands"); 1333 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1334 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1335 if (N > 1) 1336 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1337 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1338 Register ZeroReg = buildZerosVal(ResType, I); 1339 // A new virtual register to store the result struct. 1340 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1341 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1342 // Build the result name if needed. 1343 if (ResName.size() > 0) 1344 buildOpName(StructVReg, ResName, MIRBuilder); 1345 // Build the arithmetic with overflow instruction. 1346 MachineBasicBlock &BB = *I.getParent(); 1347 auto MIB = 1348 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1349 .addDef(StructVReg) 1350 .addUse(GR.getSPIRVTypeID(StructType)); 1351 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1352 MIB.addUse(I.getOperand(i).getReg()); 1353 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1354 // Build instructions to extract fields of the instruction's result. 1355 // A new virtual register to store the higher part of the result struct. 1356 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1357 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1358 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1359 auto MIB = 1360 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1361 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1362 .addUse(GR.getSPIRVTypeID(ResType)) 1363 .addUse(StructVReg) 1364 .addImm(i); 1365 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1366 } 1367 // Build boolean value from the higher part. 1368 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1369 .addDef(I.getOperand(1).getReg()) 1370 .addUse(BoolTypeReg) 1371 .addUse(HigherVReg) 1372 .addUse(ZeroReg) 1373 .constrainAllUses(TII, TRI, RBI); 1374 } 1375 1376 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1377 const SPIRVType *ResType, 1378 MachineInstr &I) const { 1379 bool Result = true; 1380 Register ScopeReg; 1381 Register MemSemEqReg; 1382 Register MemSemNeqReg; 1383 Register Ptr = I.getOperand(2).getReg(); 1384 if (!isa<GIntrinsic>(I)) { 1385 assert(I.hasOneMemOperand()); 1386 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1387 unsigned Scope = static_cast<uint32_t>(getMemScope( 1388 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1389 auto ScopeConstant = buildI32Constant(Scope, I); 1390 ScopeReg = ScopeConstant.first; 1391 Result &= ScopeConstant.second; 1392 1393 unsigned ScSem = static_cast<uint32_t>( 1394 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1395 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1396 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1397 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1398 MemSemEqReg = MemSemEqConstant.first; 1399 Result &= MemSemEqConstant.second; 1400 AtomicOrdering FO = MemOp->getFailureOrdering(); 1401 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1402 if (MemSemEq == MemSemNeq) 1403 MemSemNeqReg = MemSemEqReg; 1404 else { 1405 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1406 MemSemNeqReg = MemSemNeqConstant.first; 1407 Result &= MemSemNeqConstant.second; 1408 } 1409 } else { 1410 ScopeReg = I.getOperand(5).getReg(); 1411 MemSemEqReg = I.getOperand(6).getReg(); 1412 MemSemNeqReg = I.getOperand(7).getReg(); 1413 } 1414 1415 Register Cmp = I.getOperand(3).getReg(); 1416 Register Val = I.getOperand(4).getReg(); 1417 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1418 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1419 const DebugLoc &DL = I.getDebugLoc(); 1420 Result &= 1421 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1422 .addDef(ACmpRes) 1423 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1424 .addUse(Ptr) 1425 .addUse(ScopeReg) 1426 .addUse(MemSemEqReg) 1427 .addUse(MemSemNeqReg) 1428 .addUse(Val) 1429 .addUse(Cmp) 1430 .constrainAllUses(TII, TRI, RBI); 1431 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1432 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1433 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1434 .addDef(CmpSuccReg) 1435 .addUse(GR.getSPIRVTypeID(BoolTy)) 1436 .addUse(ACmpRes) 1437 .addUse(Cmp) 1438 .constrainAllUses(TII, TRI, RBI); 1439 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1440 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1441 .addDef(TmpReg) 1442 .addUse(GR.getSPIRVTypeID(ResType)) 1443 .addUse(ACmpRes) 1444 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1445 .addImm(0) 1446 .constrainAllUses(TII, TRI, RBI); 1447 return Result && 1448 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1449 .addDef(ResVReg) 1450 .addUse(GR.getSPIRVTypeID(ResType)) 1451 .addUse(CmpSuccReg) 1452 .addUse(TmpReg) 1453 .addImm(1) 1454 .constrainAllUses(TII, TRI, RBI); 1455 } 1456 1457 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1458 switch (SC) { 1459 case SPIRV::StorageClass::Workgroup: 1460 case SPIRV::StorageClass::CrossWorkgroup: 1461 case SPIRV::StorageClass::Function: 1462 return true; 1463 default: 1464 return false; 1465 } 1466 } 1467 1468 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1469 switch (SC) { 1470 case SPIRV::StorageClass::DeviceOnlyINTEL: 1471 case SPIRV::StorageClass::HostOnlyINTEL: 1472 return true; 1473 default: 1474 return false; 1475 } 1476 } 1477 1478 // Returns true ResVReg is referred only from global vars and OpName's. 1479 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1480 bool IsGRef = false; 1481 bool IsAllowedRefs = 1482 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1483 [&IsGRef](auto const &It) { 1484 unsigned Opcode = It.getOpcode(); 1485 if (Opcode == SPIRV::OpConstantComposite || 1486 Opcode == SPIRV::OpVariable || 1487 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1488 return IsGRef = true; 1489 return Opcode == SPIRV::OpName; 1490 }); 1491 return IsAllowedRefs && IsGRef; 1492 } 1493 1494 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1495 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1496 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1497 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1498 } 1499 1500 MachineInstrBuilder 1501 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1502 Register Src, Register DestType, 1503 uint32_t Opcode) const { 1504 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1505 TII.get(SPIRV::OpSpecConstantOp)) 1506 .addDef(Dest) 1507 .addUse(DestType) 1508 .addImm(Opcode) 1509 .addUse(Src); 1510 } 1511 1512 MachineInstrBuilder 1513 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1514 SPIRVType *SrcPtrTy) const { 1515 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1516 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1517 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1518 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1519 SPIRV::StorageClass::Generic), 1520 GR.getPointerSize())); 1521 MachineFunction *MF = I.getParent()->getParent(); 1522 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1523 MachineInstrBuilder MIB = buildSpecConstantOp( 1524 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1525 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1526 GR.add(MIB.getInstr(), MF, Tmp); 1527 return MIB; 1528 } 1529 1530 // In SPIR-V address space casting can only happen to and from the Generic 1531 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1532 // pointers to and from Generic pointers. As such, we can convert e.g. from 1533 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1534 // other combinations can only be done by a bitcast, and are probably not safe. 1535 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1536 const SPIRVType *ResType, 1537 MachineInstr &I) const { 1538 MachineBasicBlock &BB = *I.getParent(); 1539 const DebugLoc &DL = I.getDebugLoc(); 1540 1541 Register SrcPtr = I.getOperand(1).getReg(); 1542 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1543 1544 // don't generate a cast for a null that may be represented by OpTypeInt 1545 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1546 ResType->getOpcode() != SPIRV::OpTypePointer) 1547 return BuildCOPY(ResVReg, SrcPtr, I); 1548 1549 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1550 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1551 1552 if (isASCastInGVar(MRI, ResVReg)) { 1553 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1554 // are expressed by OpSpecConstantOp with an Opcode. 1555 // TODO: maybe insert a check whether the Kernel capability was declared and 1556 // so PtrCastToGeneric/GenericCastToPtr are available. 1557 unsigned SpecOpcode = 1558 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1559 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1560 : (SrcSC == SPIRV::StorageClass::Generic && 1561 isGenericCastablePtr(DstSC) 1562 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1563 : 0); 1564 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1565 // correct value of ResType and use general i8* instead. Maybe this should 1566 // be addressed in the emit-intrinsic step to infer a correct 1567 // OpConstantComposite type. 1568 if (SpecOpcode) { 1569 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1570 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1571 .constrainAllUses(TII, TRI, RBI); 1572 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1573 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1574 return MIB.constrainAllUses(TII, TRI, RBI) && 1575 buildSpecConstantOp( 1576 I, ResVReg, MIB->getOperand(0).getReg(), 1577 getUcharPtrTypeReg(I, DstSC), 1578 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1579 .constrainAllUses(TII, TRI, RBI); 1580 } 1581 } 1582 1583 // don't generate a cast between identical storage classes 1584 if (SrcSC == DstSC) 1585 return BuildCOPY(ResVReg, SrcPtr, I); 1586 1587 if ((SrcSC == SPIRV::StorageClass::Function && 1588 DstSC == SPIRV::StorageClass::Private) || 1589 (DstSC == SPIRV::StorageClass::Function && 1590 SrcSC == SPIRV::StorageClass::Private)) 1591 return BuildCOPY(ResVReg, SrcPtr, I); 1592 1593 // Casting from an eligible pointer to Generic. 1594 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1595 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1596 // Casting from Generic to an eligible pointer. 1597 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1598 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1599 // Casting between 2 eligible pointers using Generic as an intermediary. 1600 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1601 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1602 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1603 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1604 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1605 .addDef(Tmp) 1606 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1607 .addUse(SrcPtr) 1608 .constrainAllUses(TII, TRI, RBI); 1609 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1610 .addDef(ResVReg) 1611 .addUse(GR.getSPIRVTypeID(ResType)) 1612 .addUse(Tmp) 1613 .constrainAllUses(TII, TRI, RBI); 1614 } 1615 1616 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1617 // be applied 1618 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1619 return selectUnOp(ResVReg, ResType, I, 1620 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1621 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1622 return selectUnOp(ResVReg, ResType, I, 1623 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1624 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1625 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1626 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1627 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1628 1629 // Bitcast for pointers requires that the address spaces must match 1630 return false; 1631 } 1632 1633 static unsigned getFCmpOpcode(unsigned PredNum) { 1634 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1635 switch (Pred) { 1636 case CmpInst::FCMP_OEQ: 1637 return SPIRV::OpFOrdEqual; 1638 case CmpInst::FCMP_OGE: 1639 return SPIRV::OpFOrdGreaterThanEqual; 1640 case CmpInst::FCMP_OGT: 1641 return SPIRV::OpFOrdGreaterThan; 1642 case CmpInst::FCMP_OLE: 1643 return SPIRV::OpFOrdLessThanEqual; 1644 case CmpInst::FCMP_OLT: 1645 return SPIRV::OpFOrdLessThan; 1646 case CmpInst::FCMP_ONE: 1647 return SPIRV::OpFOrdNotEqual; 1648 case CmpInst::FCMP_ORD: 1649 return SPIRV::OpOrdered; 1650 case CmpInst::FCMP_UEQ: 1651 return SPIRV::OpFUnordEqual; 1652 case CmpInst::FCMP_UGE: 1653 return SPIRV::OpFUnordGreaterThanEqual; 1654 case CmpInst::FCMP_UGT: 1655 return SPIRV::OpFUnordGreaterThan; 1656 case CmpInst::FCMP_ULE: 1657 return SPIRV::OpFUnordLessThanEqual; 1658 case CmpInst::FCMP_ULT: 1659 return SPIRV::OpFUnordLessThan; 1660 case CmpInst::FCMP_UNE: 1661 return SPIRV::OpFUnordNotEqual; 1662 case CmpInst::FCMP_UNO: 1663 return SPIRV::OpUnordered; 1664 default: 1665 llvm_unreachable("Unknown predicate type for FCmp"); 1666 } 1667 } 1668 1669 static unsigned getICmpOpcode(unsigned PredNum) { 1670 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1671 switch (Pred) { 1672 case CmpInst::ICMP_EQ: 1673 return SPIRV::OpIEqual; 1674 case CmpInst::ICMP_NE: 1675 return SPIRV::OpINotEqual; 1676 case CmpInst::ICMP_SGE: 1677 return SPIRV::OpSGreaterThanEqual; 1678 case CmpInst::ICMP_SGT: 1679 return SPIRV::OpSGreaterThan; 1680 case CmpInst::ICMP_SLE: 1681 return SPIRV::OpSLessThanEqual; 1682 case CmpInst::ICMP_SLT: 1683 return SPIRV::OpSLessThan; 1684 case CmpInst::ICMP_UGE: 1685 return SPIRV::OpUGreaterThanEqual; 1686 case CmpInst::ICMP_UGT: 1687 return SPIRV::OpUGreaterThan; 1688 case CmpInst::ICMP_ULE: 1689 return SPIRV::OpULessThanEqual; 1690 case CmpInst::ICMP_ULT: 1691 return SPIRV::OpULessThan; 1692 default: 1693 llvm_unreachable("Unknown predicate type for ICmp"); 1694 } 1695 } 1696 1697 static unsigned getPtrCmpOpcode(unsigned Pred) { 1698 switch (static_cast<CmpInst::Predicate>(Pred)) { 1699 case CmpInst::ICMP_EQ: 1700 return SPIRV::OpPtrEqual; 1701 case CmpInst::ICMP_NE: 1702 return SPIRV::OpPtrNotEqual; 1703 default: 1704 llvm_unreachable("Unknown predicate type for pointer comparison"); 1705 } 1706 } 1707 1708 // Return the logical operation, or abort if none exists. 1709 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1710 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1711 switch (Pred) { 1712 case CmpInst::ICMP_EQ: 1713 return SPIRV::OpLogicalEqual; 1714 case CmpInst::ICMP_NE: 1715 return SPIRV::OpLogicalNotEqual; 1716 default: 1717 llvm_unreachable("Unknown predicate type for Bool comparison"); 1718 } 1719 } 1720 1721 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1722 if (!LLVMFloatTy) 1723 return APFloat::getZero(APFloat::IEEEsingle()); 1724 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1725 case Type::HalfTyID: 1726 return APFloat::getZero(APFloat::IEEEhalf()); 1727 default: 1728 case Type::FloatTyID: 1729 return APFloat::getZero(APFloat::IEEEsingle()); 1730 case Type::DoubleTyID: 1731 return APFloat::getZero(APFloat::IEEEdouble()); 1732 } 1733 } 1734 1735 static APFloat getOneFP(const Type *LLVMFloatTy) { 1736 if (!LLVMFloatTy) 1737 return APFloat::getOne(APFloat::IEEEsingle()); 1738 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1739 case Type::HalfTyID: 1740 return APFloat::getOne(APFloat::IEEEhalf()); 1741 default: 1742 case Type::FloatTyID: 1743 return APFloat::getOne(APFloat::IEEEsingle()); 1744 case Type::DoubleTyID: 1745 return APFloat::getOne(APFloat::IEEEdouble()); 1746 } 1747 } 1748 1749 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1750 const SPIRVType *ResType, 1751 MachineInstr &I, 1752 unsigned OpAnyOrAll) const { 1753 assert(I.getNumOperands() == 3); 1754 assert(I.getOperand(2).isReg()); 1755 MachineBasicBlock &BB = *I.getParent(); 1756 Register InputRegister = I.getOperand(2).getReg(); 1757 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1758 1759 if (!InputType) 1760 report_fatal_error("Input Type could not be determined."); 1761 1762 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1763 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1764 if (IsBoolTy && !IsVectorTy) { 1765 assert(ResVReg == I.getOperand(0).getReg()); 1766 return BuildCOPY(ResVReg, InputRegister, I); 1767 } 1768 1769 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1770 unsigned SpirvNotEqualId = 1771 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1772 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1773 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1774 Register NotEqualReg = ResVReg; 1775 1776 if (IsVectorTy) { 1777 NotEqualReg = IsBoolTy ? InputRegister 1778 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1779 const unsigned NumElts = InputType->getOperand(2).getImm(); 1780 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1781 } 1782 1783 bool Result = true; 1784 if (!IsBoolTy) { 1785 Register ConstZeroReg = 1786 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1787 1788 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1789 .addDef(NotEqualReg) 1790 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1791 .addUse(InputRegister) 1792 .addUse(ConstZeroReg) 1793 .constrainAllUses(TII, TRI, RBI); 1794 } 1795 1796 if (!IsVectorTy) 1797 return Result; 1798 1799 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1800 .addDef(ResVReg) 1801 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1802 .addUse(NotEqualReg) 1803 .constrainAllUses(TII, TRI, RBI); 1804 } 1805 1806 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1807 const SPIRVType *ResType, 1808 MachineInstr &I) const { 1809 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1810 } 1811 1812 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1813 const SPIRVType *ResType, 1814 MachineInstr &I) const { 1815 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1816 } 1817 1818 // Select the OpDot instruction for the given float dot 1819 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1820 const SPIRVType *ResType, 1821 MachineInstr &I) const { 1822 assert(I.getNumOperands() == 4); 1823 assert(I.getOperand(2).isReg()); 1824 assert(I.getOperand(3).isReg()); 1825 1826 [[maybe_unused]] SPIRVType *VecType = 1827 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1828 1829 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1830 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1831 "dot product requires a vector of at least 2 components"); 1832 1833 [[maybe_unused]] SPIRVType *EltType = 1834 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1835 1836 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1837 1838 MachineBasicBlock &BB = *I.getParent(); 1839 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1840 .addDef(ResVReg) 1841 .addUse(GR.getSPIRVTypeID(ResType)) 1842 .addUse(I.getOperand(2).getReg()) 1843 .addUse(I.getOperand(3).getReg()) 1844 .constrainAllUses(TII, TRI, RBI); 1845 } 1846 1847 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1848 const SPIRVType *ResType, 1849 MachineInstr &I, 1850 bool Signed) const { 1851 assert(I.getNumOperands() == 4); 1852 assert(I.getOperand(2).isReg()); 1853 assert(I.getOperand(3).isReg()); 1854 MachineBasicBlock &BB = *I.getParent(); 1855 1856 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1857 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1858 .addDef(ResVReg) 1859 .addUse(GR.getSPIRVTypeID(ResType)) 1860 .addUse(I.getOperand(2).getReg()) 1861 .addUse(I.getOperand(3).getReg()) 1862 .constrainAllUses(TII, TRI, RBI); 1863 } 1864 1865 // Since pre-1.6 SPIRV has no integer dot implementation, 1866 // expand by piecewise multiplying and adding the results 1867 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1868 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1869 assert(I.getNumOperands() == 4); 1870 assert(I.getOperand(2).isReg()); 1871 assert(I.getOperand(3).isReg()); 1872 MachineBasicBlock &BB = *I.getParent(); 1873 1874 // Multiply the vectors, then sum the results 1875 Register Vec0 = I.getOperand(2).getReg(); 1876 Register Vec1 = I.getOperand(3).getReg(); 1877 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1878 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1879 1880 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1881 .addDef(TmpVec) 1882 .addUse(GR.getSPIRVTypeID(VecType)) 1883 .addUse(Vec0) 1884 .addUse(Vec1) 1885 .constrainAllUses(TII, TRI, RBI); 1886 1887 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1888 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1889 "dot product requires a vector of at least 2 components"); 1890 1891 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1892 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1893 .addDef(Res) 1894 .addUse(GR.getSPIRVTypeID(ResType)) 1895 .addUse(TmpVec) 1896 .addImm(0) 1897 .constrainAllUses(TII, TRI, RBI); 1898 1899 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1900 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1901 1902 Result &= 1903 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1904 .addDef(Elt) 1905 .addUse(GR.getSPIRVTypeID(ResType)) 1906 .addUse(TmpVec) 1907 .addImm(i) 1908 .constrainAllUses(TII, TRI, RBI); 1909 1910 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1911 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1912 : ResVReg; 1913 1914 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1915 .addDef(Sum) 1916 .addUse(GR.getSPIRVTypeID(ResType)) 1917 .addUse(Res) 1918 .addUse(Elt) 1919 .constrainAllUses(TII, TRI, RBI); 1920 Res = Sum; 1921 } 1922 1923 return Result; 1924 } 1925 1926 template <bool Signed> 1927 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1928 const SPIRVType *ResType, 1929 MachineInstr &I) const { 1930 assert(I.getNumOperands() == 5); 1931 assert(I.getOperand(2).isReg()); 1932 assert(I.getOperand(3).isReg()); 1933 assert(I.getOperand(4).isReg()); 1934 MachineBasicBlock &BB = *I.getParent(); 1935 1936 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1937 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1938 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1939 .addDef(Dot) 1940 .addUse(GR.getSPIRVTypeID(ResType)) 1941 .addUse(I.getOperand(2).getReg()) 1942 .addUse(I.getOperand(3).getReg()) 1943 .constrainAllUses(TII, TRI, RBI); 1944 1945 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1946 .addDef(ResVReg) 1947 .addUse(GR.getSPIRVTypeID(ResType)) 1948 .addUse(Dot) 1949 .addUse(I.getOperand(4).getReg()) 1950 .constrainAllUses(TII, TRI, RBI); 1951 } 1952 1953 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1954 // extract the elements of the packed inputs, multiply them and add the result 1955 // to the accumulator. 1956 template <bool Signed> 1957 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1958 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1959 assert(I.getNumOperands() == 5); 1960 assert(I.getOperand(2).isReg()); 1961 assert(I.getOperand(3).isReg()); 1962 assert(I.getOperand(4).isReg()); 1963 MachineBasicBlock &BB = *I.getParent(); 1964 1965 bool Result = true; 1966 1967 // Acc = C 1968 Register Acc = I.getOperand(4).getReg(); 1969 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1970 auto ExtractOp = 1971 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1972 1973 // Extract the i8 element, multiply and add it to the accumulator 1974 for (unsigned i = 0; i < 4; i++) { 1975 // A[i] 1976 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1977 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1978 .addDef(AElt) 1979 .addUse(GR.getSPIRVTypeID(ResType)) 1980 .addUse(I.getOperand(2).getReg()) 1981 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1982 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1983 .constrainAllUses(TII, TRI, RBI); 1984 1985 // B[i] 1986 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1987 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1988 .addDef(BElt) 1989 .addUse(GR.getSPIRVTypeID(ResType)) 1990 .addUse(I.getOperand(3).getReg()) 1991 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1992 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1993 .constrainAllUses(TII, TRI, RBI); 1994 1995 // A[i] * B[i] 1996 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1997 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1998 .addDef(Mul) 1999 .addUse(GR.getSPIRVTypeID(ResType)) 2000 .addUse(AElt) 2001 .addUse(BElt) 2002 .constrainAllUses(TII, TRI, RBI); 2003 2004 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 2005 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2006 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 2007 .addDef(MaskMul) 2008 .addUse(GR.getSPIRVTypeID(ResType)) 2009 .addUse(Mul) 2010 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 2011 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 2012 .constrainAllUses(TII, TRI, RBI); 2013 2014 // Acc = Acc + A[i] * B[i] 2015 Register Sum = 2016 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 2017 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 2018 .addDef(Sum) 2019 .addUse(GR.getSPIRVTypeID(ResType)) 2020 .addUse(Acc) 2021 .addUse(MaskMul) 2022 .constrainAllUses(TII, TRI, RBI); 2023 2024 Acc = Sum; 2025 } 2026 2027 return Result; 2028 } 2029 2030 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 2031 /// does not have a saturate builtin. 2032 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 2033 const SPIRVType *ResType, 2034 MachineInstr &I) const { 2035 assert(I.getNumOperands() == 3); 2036 assert(I.getOperand(2).isReg()); 2037 MachineBasicBlock &BB = *I.getParent(); 2038 Register VZero = buildZerosValF(ResType, I); 2039 Register VOne = buildOnesValF(ResType, I); 2040 2041 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2042 .addDef(ResVReg) 2043 .addUse(GR.getSPIRVTypeID(ResType)) 2044 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2045 .addImm(GL::FClamp) 2046 .addUse(I.getOperand(2).getReg()) 2047 .addUse(VZero) 2048 .addUse(VOne) 2049 .constrainAllUses(TII, TRI, RBI); 2050 } 2051 2052 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 2053 const SPIRVType *ResType, 2054 MachineInstr &I) const { 2055 assert(I.getNumOperands() == 3); 2056 assert(I.getOperand(2).isReg()); 2057 MachineBasicBlock &BB = *I.getParent(); 2058 Register InputRegister = I.getOperand(2).getReg(); 2059 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2060 auto &DL = I.getDebugLoc(); 2061 2062 if (!InputType) 2063 report_fatal_error("Input Type could not be determined."); 2064 2065 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2066 2067 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 2068 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 2069 2070 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 2071 2072 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 2073 Register SignReg = NeedsConversion 2074 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 2075 : ResVReg; 2076 2077 bool Result = 2078 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 2079 .addDef(SignReg) 2080 .addUse(GR.getSPIRVTypeID(InputType)) 2081 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2082 .addImm(SignOpcode) 2083 .addUse(InputRegister) 2084 .constrainAllUses(TII, TRI, RBI); 2085 2086 if (NeedsConversion) { 2087 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2088 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2089 .addDef(ResVReg) 2090 .addUse(GR.getSPIRVTypeID(ResType)) 2091 .addUse(SignReg) 2092 .constrainAllUses(TII, TRI, RBI); 2093 } 2094 2095 return Result; 2096 } 2097 2098 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2099 const SPIRVType *ResType, 2100 MachineInstr &I, 2101 unsigned Opcode) const { 2102 MachineBasicBlock &BB = *I.getParent(); 2103 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2104 2105 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2106 .addDef(ResVReg) 2107 .addUse(GR.getSPIRVTypeID(ResType)) 2108 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2109 IntTy, TII)); 2110 2111 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2112 BMI.addUse(I.getOperand(J).getReg()); 2113 } 2114 2115 return BMI.constrainAllUses(TII, TRI, RBI); 2116 } 2117 2118 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2119 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2120 2121 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2122 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2123 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2124 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2125 SPIRV::OpGroupNonUniformBallot); 2126 2127 MachineBasicBlock &BB = *I.getParent(); 2128 Result &= 2129 BuildMI(BB, I, I.getDebugLoc(), 2130 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2131 .addDef(ResVReg) 2132 .addUse(GR.getSPIRVTypeID(ResType)) 2133 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2134 .addImm(SPIRV::GroupOperation::Reduce) 2135 .addUse(BallotReg) 2136 .constrainAllUses(TII, TRI, RBI); 2137 2138 return Result; 2139 } 2140 2141 bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg, 2142 const SPIRVType *ResType, 2143 MachineInstr &I) const { 2144 assert(I.getNumOperands() == 3); 2145 assert(I.getOperand(2).isReg()); 2146 MachineBasicBlock &BB = *I.getParent(); 2147 Register InputRegister = I.getOperand(2).getReg(); 2148 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2149 2150 if (!InputType) 2151 report_fatal_error("Input Type could not be determined."); 2152 2153 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2154 // Retreive the operation to use based on input type 2155 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2156 auto Opcode = 2157 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd : SPIRV::OpGroupNonUniformIAdd; 2158 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2159 .addDef(ResVReg) 2160 .addUse(GR.getSPIRVTypeID(ResType)) 2161 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2162 .addImm(SPIRV::GroupOperation::Reduce) 2163 .addUse(I.getOperand(2).getReg()); 2164 } 2165 2166 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2167 const SPIRVType *ResType, 2168 MachineInstr &I) const { 2169 MachineBasicBlock &BB = *I.getParent(); 2170 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2171 .addDef(ResVReg) 2172 .addUse(GR.getSPIRVTypeID(ResType)) 2173 .addUse(I.getOperand(1).getReg()) 2174 .constrainAllUses(TII, TRI, RBI); 2175 } 2176 2177 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2178 const SPIRVType *ResType, 2179 MachineInstr &I) const { 2180 // There is no way to implement `freeze` correctly without support on SPIR-V 2181 // standard side, but we may at least address a simple (static) case when 2182 // undef/poison value presence is obvious. The main benefit of even 2183 // incomplete `freeze` support is preventing of translation from crashing due 2184 // to lack of support on legalization and instruction selection steps. 2185 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2186 return false; 2187 Register OpReg = I.getOperand(1).getReg(); 2188 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2189 Register Reg; 2190 switch (Def->getOpcode()) { 2191 case SPIRV::ASSIGN_TYPE: 2192 if (MachineInstr *AssignToDef = 2193 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2194 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2195 Reg = Def->getOperand(2).getReg(); 2196 } 2197 break; 2198 case SPIRV::OpUndef: 2199 Reg = Def->getOperand(1).getReg(); 2200 break; 2201 } 2202 unsigned DestOpCode; 2203 if (Reg.isValid()) { 2204 DestOpCode = SPIRV::OpConstantNull; 2205 } else { 2206 DestOpCode = TargetOpcode::COPY; 2207 Reg = OpReg; 2208 } 2209 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2210 .addDef(I.getOperand(0).getReg()) 2211 .addUse(Reg) 2212 .constrainAllUses(TII, TRI, RBI); 2213 } 2214 return false; 2215 } 2216 2217 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2218 const SPIRVType *ResType) { 2219 Register OpReg = ResType->getOperand(2).getReg(); 2220 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2221 if (!OpDef) 2222 return 0; 2223 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2224 OpDef->getOperand(1).isReg()) { 2225 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2226 OpDef = RefDef; 2227 } 2228 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2229 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2230 : 0; 2231 return N; 2232 } 2233 2234 // Return true if the type represents a constant register 2235 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2236 SmallPtrSet<SPIRVType *, 4> &Visited) { 2237 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2238 OpDef->getOperand(1).isReg()) { 2239 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2240 OpDef = RefDef; 2241 } 2242 2243 if (Visited.contains(OpDef)) 2244 return true; 2245 Visited.insert(OpDef); 2246 2247 unsigned Opcode = OpDef->getOpcode(); 2248 switch (Opcode) { 2249 case TargetOpcode::G_CONSTANT: 2250 case TargetOpcode::G_FCONSTANT: 2251 return true; 2252 case TargetOpcode::G_INTRINSIC: 2253 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2254 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2255 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2256 Intrinsic::spv_const_composite; 2257 case TargetOpcode::G_BUILD_VECTOR: 2258 case TargetOpcode::G_SPLAT_VECTOR: { 2259 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2260 i++) { 2261 SPIRVType *OpNestedDef = 2262 OpDef->getOperand(i).isReg() 2263 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2264 : nullptr; 2265 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2266 return false; 2267 } 2268 return true; 2269 } 2270 } 2271 return false; 2272 } 2273 2274 // Return true if the virtual register represents a constant 2275 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2276 SmallPtrSet<SPIRVType *, 4> Visited; 2277 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2278 return isConstReg(MRI, OpDef, Visited); 2279 return false; 2280 } 2281 2282 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2283 const SPIRVType *ResType, 2284 MachineInstr &I) const { 2285 unsigned N = 0; 2286 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2287 N = GR.getScalarOrVectorComponentCount(ResType); 2288 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2289 N = getArrayComponentCount(MRI, ResType); 2290 else 2291 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2292 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2293 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2294 2295 // check if we may construct a constant vector 2296 bool IsConst = true; 2297 for (unsigned i = I.getNumExplicitDefs(); 2298 i < I.getNumExplicitOperands() && IsConst; ++i) 2299 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2300 IsConst = false; 2301 2302 if (!IsConst && N < 2) 2303 report_fatal_error( 2304 "There must be at least two constituent operands in a vector"); 2305 2306 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2307 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2308 TII.get(IsConst ? SPIRV::OpConstantComposite 2309 : SPIRV::OpCompositeConstruct)) 2310 .addDef(ResVReg) 2311 .addUse(GR.getSPIRVTypeID(ResType)); 2312 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2313 MIB.addUse(I.getOperand(i).getReg()); 2314 return MIB.constrainAllUses(TII, TRI, RBI); 2315 } 2316 2317 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2318 const SPIRVType *ResType, 2319 MachineInstr &I) const { 2320 unsigned N = 0; 2321 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2322 N = GR.getScalarOrVectorComponentCount(ResType); 2323 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2324 N = getArrayComponentCount(MRI, ResType); 2325 else 2326 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2327 2328 unsigned OpIdx = I.getNumExplicitDefs(); 2329 if (!I.getOperand(OpIdx).isReg()) 2330 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2331 2332 // check if we may construct a constant vector 2333 Register OpReg = I.getOperand(OpIdx).getReg(); 2334 bool IsConst = isConstReg(MRI, OpReg); 2335 2336 if (!IsConst && N < 2) 2337 report_fatal_error( 2338 "There must be at least two constituent operands in a vector"); 2339 2340 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2341 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2342 TII.get(IsConst ? SPIRV::OpConstantComposite 2343 : SPIRV::OpCompositeConstruct)) 2344 .addDef(ResVReg) 2345 .addUse(GR.getSPIRVTypeID(ResType)); 2346 for (unsigned i = 0; i < N; ++i) 2347 MIB.addUse(OpReg); 2348 return MIB.constrainAllUses(TII, TRI, RBI); 2349 } 2350 2351 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2352 const SPIRVType *ResType, 2353 MachineInstr &I) const { 2354 2355 unsigned Opcode; 2356 2357 if (STI.canUseExtension( 2358 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2359 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2360 Opcode = SPIRV::OpDemoteToHelperInvocation; 2361 } else { 2362 Opcode = SPIRV::OpKill; 2363 // OpKill must be the last operation of any basic block. 2364 if (MachineInstr *NextI = I.getNextNode()) { 2365 GR.invalidateMachineInstr(NextI); 2366 NextI->removeFromParent(); 2367 } 2368 } 2369 2370 MachineBasicBlock &BB = *I.getParent(); 2371 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2372 .constrainAllUses(TII, TRI, RBI); 2373 } 2374 2375 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2376 const SPIRVType *ResType, 2377 unsigned CmpOpc, 2378 MachineInstr &I) const { 2379 Register Cmp0 = I.getOperand(2).getReg(); 2380 Register Cmp1 = I.getOperand(3).getReg(); 2381 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2382 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2383 "CMP operands should have the same type"); 2384 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2385 .addDef(ResVReg) 2386 .addUse(GR.getSPIRVTypeID(ResType)) 2387 .addUse(Cmp0) 2388 .addUse(Cmp1) 2389 .constrainAllUses(TII, TRI, RBI); 2390 } 2391 2392 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2393 const SPIRVType *ResType, 2394 MachineInstr &I) const { 2395 auto Pred = I.getOperand(1).getPredicate(); 2396 unsigned CmpOpc; 2397 2398 Register CmpOperand = I.getOperand(2).getReg(); 2399 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2400 CmpOpc = getPtrCmpOpcode(Pred); 2401 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2402 CmpOpc = getBoolCmpOpcode(Pred); 2403 else 2404 CmpOpc = getICmpOpcode(Pred); 2405 return selectCmp(ResVReg, ResType, CmpOpc, I); 2406 } 2407 2408 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2409 const MachineInstr &I, 2410 int OpIdx) const { 2411 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2412 "Expected G_FCONSTANT"); 2413 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2414 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2415 } 2416 2417 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2418 const MachineInstr &I, 2419 int OpIdx) const { 2420 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2421 "Expected G_CONSTANT"); 2422 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2423 } 2424 2425 std::pair<Register, bool> 2426 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2427 const SPIRVType *ResType) const { 2428 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2429 const SPIRVType *SpvI32Ty = 2430 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2431 // Find a constant in DT or build a new one. 2432 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2433 Register NewReg = GR.find(ConstInt, GR.CurMF); 2434 bool Result = true; 2435 if (!NewReg.isValid()) { 2436 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2437 GR.add(ConstInt, GR.CurMF, NewReg); 2438 MachineInstr *MI; 2439 MachineBasicBlock &BB = *I.getParent(); 2440 if (Val == 0) { 2441 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2442 .addDef(NewReg) 2443 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2444 } else { 2445 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2446 .addDef(NewReg) 2447 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2448 .addImm(APInt(32, Val).getZExtValue()); 2449 } 2450 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2451 } 2452 return {NewReg, Result}; 2453 } 2454 2455 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2456 const SPIRVType *ResType, 2457 MachineInstr &I) const { 2458 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2459 return selectCmp(ResVReg, ResType, CmpOp, I); 2460 } 2461 2462 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2463 MachineInstr &I) const { 2464 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2465 bool ZeroAsNull = STI.isOpenCLEnv(); 2466 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2467 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2468 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2469 } 2470 2471 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2472 MachineInstr &I) const { 2473 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2474 bool ZeroAsNull = STI.isOpenCLEnv(); 2475 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2476 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2477 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2478 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2479 } 2480 2481 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2482 MachineInstr &I) const { 2483 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2484 bool ZeroAsNull = STI.isOpenCLEnv(); 2485 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2486 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2487 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2488 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2489 } 2490 2491 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2492 const SPIRVType *ResType, 2493 MachineInstr &I) const { 2494 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2495 APInt One = 2496 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2497 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2498 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2499 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2500 } 2501 2502 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2503 const SPIRVType *ResType, 2504 MachineInstr &I, 2505 bool IsSigned) const { 2506 // To extend a bool, we need to use OpSelect between constants. 2507 Register ZeroReg = buildZerosVal(ResType, I); 2508 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2509 bool IsScalarBool = 2510 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2511 unsigned Opcode = 2512 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2513 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2514 .addDef(ResVReg) 2515 .addUse(GR.getSPIRVTypeID(ResType)) 2516 .addUse(I.getOperand(1).getReg()) 2517 .addUse(OneReg) 2518 .addUse(ZeroReg) 2519 .constrainAllUses(TII, TRI, RBI); 2520 } 2521 2522 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2523 const SPIRVType *ResType, 2524 MachineInstr &I, bool IsSigned, 2525 unsigned Opcode) const { 2526 Register SrcReg = I.getOperand(1).getReg(); 2527 // We can convert bool value directly to float type without OpConvert*ToF, 2528 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2529 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2530 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2531 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2532 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2533 const unsigned NumElts = ResType->getOperand(2).getImm(); 2534 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2535 } 2536 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2537 selectSelect(SrcReg, TmpType, I, false); 2538 } 2539 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2540 } 2541 2542 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2543 const SPIRVType *ResType, 2544 MachineInstr &I, bool IsSigned) const { 2545 Register SrcReg = I.getOperand(1).getReg(); 2546 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2547 return selectSelect(ResVReg, ResType, I, IsSigned); 2548 2549 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2550 if (SrcType == ResType) 2551 return BuildCOPY(ResVReg, SrcReg, I); 2552 2553 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2554 return selectUnOp(ResVReg, ResType, I, Opcode); 2555 } 2556 2557 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2558 const SPIRVType *ResType, 2559 MachineInstr &I, 2560 bool IsSigned) const { 2561 MachineIRBuilder MIRBuilder(I); 2562 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2563 MachineBasicBlock &BB = *I.getParent(); 2564 // Ensure we have bool. 2565 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2566 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2567 if (N > 1) 2568 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2569 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2570 // Build less-than-equal and less-than. 2571 // TODO: replace with one-liner createVirtualRegister() from 2572 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2573 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2574 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2575 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2576 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2577 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2578 : SPIRV::OpULessThanEqual)) 2579 .addDef(IsLessEqReg) 2580 .addUse(BoolTypeReg) 2581 .addUse(I.getOperand(1).getReg()) 2582 .addUse(I.getOperand(2).getReg()) 2583 .constrainAllUses(TII, TRI, RBI); 2584 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2585 MRI->setType(IsLessReg, LLT::scalar(64)); 2586 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2587 Result &= BuildMI(BB, I, I.getDebugLoc(), 2588 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2589 .addDef(IsLessReg) 2590 .addUse(BoolTypeReg) 2591 .addUse(I.getOperand(1).getReg()) 2592 .addUse(I.getOperand(2).getReg()) 2593 .constrainAllUses(TII, TRI, RBI); 2594 // Build selects. 2595 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2596 Register NegOneOrZeroReg = 2597 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2598 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2599 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2600 unsigned SelectOpcode = 2601 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2602 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2603 .addDef(NegOneOrZeroReg) 2604 .addUse(ResTypeReg) 2605 .addUse(IsLessReg) 2606 .addUse(buildOnesVal(true, ResType, I)) // -1 2607 .addUse(buildZerosVal(ResType, I)) 2608 .constrainAllUses(TII, TRI, RBI); 2609 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2610 .addDef(ResVReg) 2611 .addUse(ResTypeReg) 2612 .addUse(IsLessEqReg) 2613 .addUse(NegOneOrZeroReg) // -1 or 0 2614 .addUse(buildOnesVal(false, ResType, I)) 2615 .constrainAllUses(TII, TRI, RBI); 2616 } 2617 2618 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2619 Register ResVReg, 2620 MachineInstr &I, 2621 const SPIRVType *IntTy, 2622 const SPIRVType *BoolTy) const { 2623 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2624 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2625 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2626 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2627 Register Zero = buildZerosVal(IntTy, I); 2628 Register One = buildOnesVal(false, IntTy, I); 2629 MachineBasicBlock &BB = *I.getParent(); 2630 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2631 .addDef(BitIntReg) 2632 .addUse(GR.getSPIRVTypeID(IntTy)) 2633 .addUse(IntReg) 2634 .addUse(One) 2635 .constrainAllUses(TII, TRI, RBI); 2636 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2637 .addDef(ResVReg) 2638 .addUse(GR.getSPIRVTypeID(BoolTy)) 2639 .addUse(BitIntReg) 2640 .addUse(Zero) 2641 .constrainAllUses(TII, TRI, RBI); 2642 } 2643 2644 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2645 const SPIRVType *ResType, 2646 MachineInstr &I) const { 2647 Register IntReg = I.getOperand(1).getReg(); 2648 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2649 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2650 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2651 if (ArgType == ResType) 2652 return BuildCOPY(ResVReg, IntReg, I); 2653 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2654 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2655 return selectUnOp(ResVReg, ResType, I, Opcode); 2656 } 2657 2658 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2659 const SPIRVType *ResType, 2660 const APInt &Imm, 2661 MachineInstr &I) const { 2662 unsigned TyOpcode = ResType->getOpcode(); 2663 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2664 MachineBasicBlock &BB = *I.getParent(); 2665 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2666 Imm.isZero()) 2667 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2668 .addDef(ResVReg) 2669 .addUse(GR.getSPIRVTypeID(ResType)) 2670 .constrainAllUses(TII, TRI, RBI); 2671 if (TyOpcode == SPIRV::OpTypeInt) { 2672 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2673 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2674 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2675 } 2676 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2677 .addDef(ResVReg) 2678 .addUse(GR.getSPIRVTypeID(ResType)); 2679 // <=32-bit integers should be caught by the sdag pattern. 2680 assert(Imm.getBitWidth() > 32); 2681 addNumImm(Imm, MIB); 2682 return MIB.constrainAllUses(TII, TRI, RBI); 2683 } 2684 2685 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2686 const SPIRVType *ResType, 2687 MachineInstr &I) const { 2688 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2689 .addDef(ResVReg) 2690 .addUse(GR.getSPIRVTypeID(ResType)) 2691 .constrainAllUses(TII, TRI, RBI); 2692 } 2693 2694 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2695 assert(MO.isReg()); 2696 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2697 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2698 assert(TypeInst->getOperand(1).isReg()); 2699 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2700 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2701 } 2702 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2703 } 2704 2705 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2706 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2707 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2708 return TypeInst->getOperand(2).getImm(); 2709 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2710 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2711 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2712 } 2713 2714 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2715 const SPIRVType *ResType, 2716 MachineInstr &I) const { 2717 MachineBasicBlock &BB = *I.getParent(); 2718 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2719 .addDef(ResVReg) 2720 .addUse(GR.getSPIRVTypeID(ResType)) 2721 // object to insert 2722 .addUse(I.getOperand(3).getReg()) 2723 // composite to insert into 2724 .addUse(I.getOperand(2).getReg()); 2725 for (unsigned i = 4; i < I.getNumOperands(); i++) 2726 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2727 return MIB.constrainAllUses(TII, TRI, RBI); 2728 } 2729 2730 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2731 const SPIRVType *ResType, 2732 MachineInstr &I) const { 2733 MachineBasicBlock &BB = *I.getParent(); 2734 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2735 .addDef(ResVReg) 2736 .addUse(GR.getSPIRVTypeID(ResType)) 2737 .addUse(I.getOperand(2).getReg()); 2738 for (unsigned i = 3; i < I.getNumOperands(); i++) 2739 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2740 return MIB.constrainAllUses(TII, TRI, RBI); 2741 } 2742 2743 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2744 const SPIRVType *ResType, 2745 MachineInstr &I) const { 2746 if (isImm(I.getOperand(4), MRI)) 2747 return selectInsertVal(ResVReg, ResType, I); 2748 MachineBasicBlock &BB = *I.getParent(); 2749 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2750 .addDef(ResVReg) 2751 .addUse(GR.getSPIRVTypeID(ResType)) 2752 .addUse(I.getOperand(2).getReg()) 2753 .addUse(I.getOperand(3).getReg()) 2754 .addUse(I.getOperand(4).getReg()) 2755 .constrainAllUses(TII, TRI, RBI); 2756 } 2757 2758 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2759 const SPIRVType *ResType, 2760 MachineInstr &I) const { 2761 if (isImm(I.getOperand(3), MRI)) 2762 return selectExtractVal(ResVReg, ResType, I); 2763 MachineBasicBlock &BB = *I.getParent(); 2764 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2765 .addDef(ResVReg) 2766 .addUse(GR.getSPIRVTypeID(ResType)) 2767 .addUse(I.getOperand(2).getReg()) 2768 .addUse(I.getOperand(3).getReg()) 2769 .constrainAllUses(TII, TRI, RBI); 2770 } 2771 2772 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2773 const SPIRVType *ResType, 2774 MachineInstr &I) const { 2775 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2776 2777 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2778 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2779 // we have to use Op[InBounds]AccessChain. 2780 const unsigned Opcode = STI.isVulkanEnv() 2781 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2782 : SPIRV::OpAccessChain) 2783 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2784 : SPIRV::OpPtrAccessChain); 2785 2786 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2787 .addDef(ResVReg) 2788 .addUse(GR.getSPIRVTypeID(ResType)) 2789 // Object to get a pointer to. 2790 .addUse(I.getOperand(3).getReg()); 2791 // Adding indices. 2792 const unsigned StartingIndex = 2793 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2794 ? 5 2795 : 4; 2796 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2797 Res.addUse(I.getOperand(i).getReg()); 2798 return Res.constrainAllUses(TII, TRI, RBI); 2799 } 2800 2801 // Maybe wrap a value into OpSpecConstantOp 2802 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2803 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2804 bool Result = true; 2805 unsigned Lim = I.getNumExplicitOperands(); 2806 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2807 Register OpReg = I.getOperand(i).getReg(); 2808 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2809 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2810 SmallPtrSet<SPIRVType *, 4> Visited; 2811 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2812 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2813 GR.isAggregateType(OpType)) { 2814 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2815 // by selectAddrSpaceCast() 2816 CompositeArgs.push_back(OpReg); 2817 continue; 2818 } 2819 MachineFunction *MF = I.getMF(); 2820 Register WrapReg = GR.find(OpDefine, MF); 2821 if (WrapReg.isValid()) { 2822 CompositeArgs.push_back(WrapReg); 2823 continue; 2824 } 2825 // Create a new register for the wrapper 2826 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2827 GR.add(OpDefine, MF, WrapReg); 2828 CompositeArgs.push_back(WrapReg); 2829 // Decorate the wrapper register and generate a new instruction 2830 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2831 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2832 MachineBasicBlock &BB = *I.getParent(); 2833 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2834 .addDef(WrapReg) 2835 .addUse(GR.getSPIRVTypeID(OpType)) 2836 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2837 .addUse(OpReg) 2838 .constrainAllUses(TII, TRI, RBI); 2839 if (!Result) 2840 break; 2841 } 2842 return Result; 2843 } 2844 2845 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2846 const SPIRVType *ResType, 2847 MachineInstr &I) const { 2848 MachineBasicBlock &BB = *I.getParent(); 2849 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2850 switch (IID) { 2851 case Intrinsic::spv_load: 2852 return selectLoad(ResVReg, ResType, I); 2853 case Intrinsic::spv_store: 2854 return selectStore(I); 2855 case Intrinsic::spv_extractv: 2856 return selectExtractVal(ResVReg, ResType, I); 2857 case Intrinsic::spv_insertv: 2858 return selectInsertVal(ResVReg, ResType, I); 2859 case Intrinsic::spv_extractelt: 2860 return selectExtractElt(ResVReg, ResType, I); 2861 case Intrinsic::spv_insertelt: 2862 return selectInsertElt(ResVReg, ResType, I); 2863 case Intrinsic::spv_gep: 2864 return selectGEP(ResVReg, ResType, I); 2865 case Intrinsic::spv_unref_global: 2866 case Intrinsic::spv_init_global: { 2867 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2868 MachineInstr *Init = I.getNumExplicitOperands() > 2 2869 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2870 : nullptr; 2871 assert(MI); 2872 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2873 } 2874 case Intrinsic::spv_undef: { 2875 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2876 .addDef(ResVReg) 2877 .addUse(GR.getSPIRVTypeID(ResType)); 2878 return MIB.constrainAllUses(TII, TRI, RBI); 2879 } 2880 case Intrinsic::spv_const_composite: { 2881 // If no values are attached, the composite is null constant. 2882 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2883 // Select a proper instruction. 2884 unsigned Opcode = SPIRV::OpConstantNull; 2885 SmallVector<Register> CompositeArgs; 2886 if (!IsNull) { 2887 Opcode = SPIRV::OpConstantComposite; 2888 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2889 return false; 2890 } 2891 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2892 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2893 .addDef(ResVReg) 2894 .addUse(GR.getSPIRVTypeID(ResType)); 2895 // skip type MD node we already used when generated assign.type for this 2896 if (!IsNull) { 2897 for (Register OpReg : CompositeArgs) 2898 MIB.addUse(OpReg); 2899 } 2900 return MIB.constrainAllUses(TII, TRI, RBI); 2901 } 2902 case Intrinsic::spv_assign_name: { 2903 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2904 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2905 for (unsigned i = I.getNumExplicitDefs() + 2; 2906 i < I.getNumExplicitOperands(); ++i) { 2907 MIB.addImm(I.getOperand(i).getImm()); 2908 } 2909 return MIB.constrainAllUses(TII, TRI, RBI); 2910 } 2911 case Intrinsic::spv_switch: { 2912 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2913 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2914 if (I.getOperand(i).isReg()) 2915 MIB.addReg(I.getOperand(i).getReg()); 2916 else if (I.getOperand(i).isCImm()) 2917 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2918 else if (I.getOperand(i).isMBB()) 2919 MIB.addMBB(I.getOperand(i).getMBB()); 2920 else 2921 llvm_unreachable("Unexpected OpSwitch operand"); 2922 } 2923 return MIB.constrainAllUses(TII, TRI, RBI); 2924 } 2925 case Intrinsic::spv_loop_merge: { 2926 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge)); 2927 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2928 assert(I.getOperand(i).isMBB()); 2929 MIB.addMBB(I.getOperand(i).getMBB()); 2930 } 2931 MIB.addImm(SPIRV::SelectionControl::None); 2932 return MIB.constrainAllUses(TII, TRI, RBI); 2933 } 2934 case Intrinsic::spv_selection_merge: { 2935 auto MIB = 2936 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge)); 2937 assert(I.getOperand(1).isMBB() && 2938 "operand 1 to spv_selection_merge must be a basic block"); 2939 MIB.addMBB(I.getOperand(1).getMBB()); 2940 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm())); 2941 return MIB.constrainAllUses(TII, TRI, RBI); 2942 } 2943 case Intrinsic::spv_cmpxchg: 2944 return selectAtomicCmpXchg(ResVReg, ResType, I); 2945 case Intrinsic::spv_unreachable: 2946 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2947 .constrainAllUses(TII, TRI, RBI); 2948 case Intrinsic::spv_alloca: 2949 return selectFrameIndex(ResVReg, ResType, I); 2950 case Intrinsic::spv_alloca_array: 2951 return selectAllocaArray(ResVReg, ResType, I); 2952 case Intrinsic::spv_assume: 2953 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2954 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2955 .addUse(I.getOperand(1).getReg()) 2956 .constrainAllUses(TII, TRI, RBI); 2957 break; 2958 case Intrinsic::spv_expect: 2959 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2960 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2961 .addDef(ResVReg) 2962 .addUse(GR.getSPIRVTypeID(ResType)) 2963 .addUse(I.getOperand(2).getReg()) 2964 .addUse(I.getOperand(3).getReg()) 2965 .constrainAllUses(TII, TRI, RBI); 2966 break; 2967 case Intrinsic::arithmetic_fence: 2968 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2969 return BuildMI(BB, I, I.getDebugLoc(), 2970 TII.get(SPIRV::OpArithmeticFenceEXT)) 2971 .addDef(ResVReg) 2972 .addUse(GR.getSPIRVTypeID(ResType)) 2973 .addUse(I.getOperand(2).getReg()) 2974 .constrainAllUses(TII, TRI, RBI); 2975 else 2976 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2977 break; 2978 case Intrinsic::spv_thread_id: 2979 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 2980 // intrinsic in LLVM IR for SPIR-V backend. 2981 // 2982 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 2983 // `GlobalInvocationId` builtin variable 2984 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 2985 ResType, I); 2986 case Intrinsic::spv_thread_id_in_group: 2987 // The HLSL SV_GroupThreadId semantic is lowered to 2988 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 2989 // 2990 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 2991 // translated to a `LocalInvocationId` builtin variable 2992 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 2993 ResType, I); 2994 case Intrinsic::spv_group_id: 2995 // The HLSL SV_GroupId semantic is lowered to 2996 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. 2997 // 2998 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` 2999 // builtin variable 3000 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, 3001 I); 3002 case Intrinsic::spv_fdot: 3003 return selectFloatDot(ResVReg, ResType, I); 3004 case Intrinsic::spv_udot: 3005 case Intrinsic::spv_sdot: 3006 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3007 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3008 return selectIntegerDot(ResVReg, ResType, I, 3009 /*Signed=*/IID == Intrinsic::spv_sdot); 3010 return selectIntegerDotExpansion(ResVReg, ResType, I); 3011 case Intrinsic::spv_dot4add_i8packed: 3012 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3013 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3014 return selectDot4AddPacked<true>(ResVReg, ResType, I); 3015 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 3016 case Intrinsic::spv_dot4add_u8packed: 3017 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3018 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3019 return selectDot4AddPacked<false>(ResVReg, ResType, I); 3020 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 3021 case Intrinsic::spv_all: 3022 return selectAll(ResVReg, ResType, I); 3023 case Intrinsic::spv_any: 3024 return selectAny(ResVReg, ResType, I); 3025 case Intrinsic::spv_cross: 3026 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 3027 case Intrinsic::spv_distance: 3028 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); 3029 case Intrinsic::spv_lerp: 3030 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 3031 case Intrinsic::spv_length: 3032 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 3033 case Intrinsic::spv_degrees: 3034 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 3035 case Intrinsic::spv_frac: 3036 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 3037 case Intrinsic::spv_normalize: 3038 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 3039 case Intrinsic::spv_reflect: 3040 return selectExtInst(ResVReg, ResType, I, GL::Reflect); 3041 case Intrinsic::spv_rsqrt: 3042 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 3043 case Intrinsic::spv_sign: 3044 return selectSign(ResVReg, ResType, I); 3045 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 3046 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 3047 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 3048 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 3049 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb 3050 return selectFirstBitLow(ResVReg, ResType, I); 3051 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 3052 bool Result = true; 3053 auto MemSemConstant = 3054 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 3055 Register MemSemReg = MemSemConstant.first; 3056 Result &= MemSemConstant.second; 3057 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 3058 Register ScopeReg = ScopeConstant.first; 3059 Result &= ScopeConstant.second; 3060 MachineBasicBlock &BB = *I.getParent(); 3061 return Result && 3062 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 3063 .addUse(ScopeReg) 3064 .addUse(ScopeReg) 3065 .addUse(MemSemReg) 3066 .constrainAllUses(TII, TRI, RBI); 3067 } 3068 case Intrinsic::spv_lifetime_start: 3069 case Intrinsic::spv_lifetime_end: { 3070 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 3071 : SPIRV::OpLifetimeStop; 3072 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 3073 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 3074 if (Size == -1) 3075 Size = 0; 3076 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 3077 .addUse(PtrReg) 3078 .addImm(Size) 3079 .constrainAllUses(TII, TRI, RBI); 3080 } 3081 case Intrinsic::spv_saturate: 3082 return selectSaturate(ResVReg, ResType, I); 3083 case Intrinsic::spv_nclamp: 3084 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 3085 case Intrinsic::spv_uclamp: 3086 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 3087 case Intrinsic::spv_sclamp: 3088 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 3089 case Intrinsic::spv_wave_active_countbits: 3090 return selectWaveActiveCountBits(ResVReg, ResType, I); 3091 case Intrinsic::spv_wave_all: 3092 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 3093 case Intrinsic::spv_wave_any: 3094 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 3095 case Intrinsic::spv_wave_is_first_lane: 3096 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 3097 case Intrinsic::spv_wave_reduce_sum: 3098 return selectWaveReduceSum(ResVReg, ResType, I); 3099 case Intrinsic::spv_wave_readlane: 3100 return selectWaveOpInst(ResVReg, ResType, I, 3101 SPIRV::OpGroupNonUniformShuffle); 3102 case Intrinsic::spv_step: 3103 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 3104 case Intrinsic::spv_radians: 3105 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 3106 // Discard intrinsics which we do not expect to actually represent code after 3107 // lowering or intrinsics which are not implemented but should not crash when 3108 // found in a customer's LLVM IR input. 3109 case Intrinsic::instrprof_increment: 3110 case Intrinsic::instrprof_increment_step: 3111 case Intrinsic::instrprof_value_profile: 3112 break; 3113 // Discard internal intrinsics. 3114 case Intrinsic::spv_value_md: 3115 break; 3116 case Intrinsic::spv_resource_handlefrombinding: { 3117 return selectHandleFromBinding(ResVReg, ResType, I); 3118 } 3119 case Intrinsic::spv_resource_store_typedbuffer: { 3120 return selectImageWriteIntrinsic(I); 3121 } 3122 case Intrinsic::spv_resource_load_typedbuffer: { 3123 return selectReadImageIntrinsic(ResVReg, ResType, I); 3124 } 3125 case Intrinsic::spv_resource_getpointer: { 3126 return selectResourceGetPointer(ResVReg, ResType, I); 3127 } 3128 case Intrinsic::spv_discard: { 3129 return selectDiscard(ResVReg, ResType, I); 3130 } 3131 default: { 3132 std::string DiagMsg; 3133 raw_string_ostream OS(DiagMsg); 3134 I.print(OS); 3135 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3136 report_fatal_error(DiagMsg.c_str(), false); 3137 } 3138 } 3139 return true; 3140 } 3141 3142 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3143 const SPIRVType *ResType, 3144 MachineInstr &I) const { 3145 return true; 3146 } 3147 3148 bool SPIRVInstructionSelector::selectReadImageIntrinsic( 3149 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3150 3151 // If the load of the image is in a different basic block, then 3152 // this will generate invalid code. A proper solution is to move 3153 // the OpLoad from selectHandleFromBinding here. However, to do 3154 // that we will need to change the return type of the intrinsic. 3155 // We will do that when we can, but for now trying to move forward with other 3156 // issues. 3157 Register ImageReg = I.getOperand(2).getReg(); 3158 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 3159 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 3160 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 3161 *ImageDef, I)) { 3162 return false; 3163 } 3164 3165 Register IdxReg = I.getOperand(3).getReg(); 3166 DebugLoc Loc = I.getDebugLoc(); 3167 MachineInstr &Pos = I; 3168 3169 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, Loc, Pos); 3170 } 3171 3172 bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg, 3173 const SPIRVType *ResType, 3174 Register ImageReg, 3175 Register IdxReg, DebugLoc Loc, 3176 MachineInstr &Pos) const { 3177 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3178 if (ResultSize == 4) { 3179 return BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead)) 3180 .addDef(ResVReg) 3181 .addUse(GR.getSPIRVTypeID(ResType)) 3182 .addUse(ImageReg) 3183 .addUse(IdxReg) 3184 .constrainAllUses(TII, TRI, RBI); 3185 } 3186 3187 SPIRVType *ReadType = widenTypeToVec4(ResType, Pos); 3188 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3189 bool Succeed = 3190 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead)) 3191 .addDef(ReadReg) 3192 .addUse(GR.getSPIRVTypeID(ReadType)) 3193 .addUse(ImageReg) 3194 .addUse(IdxReg) 3195 .constrainAllUses(TII, TRI, RBI); 3196 if (!Succeed) 3197 return false; 3198 3199 if (ResultSize == 1) { 3200 return BuildMI(*Pos.getParent(), Pos, Loc, 3201 TII.get(SPIRV::OpCompositeExtract)) 3202 .addDef(ResVReg) 3203 .addUse(GR.getSPIRVTypeID(ResType)) 3204 .addUse(ReadReg) 3205 .addImm(0) 3206 .constrainAllUses(TII, TRI, RBI); 3207 } 3208 return extractSubvector(ResVReg, ResType, ReadReg, Pos); 3209 } 3210 3211 bool SPIRVInstructionSelector::selectResourceGetPointer( 3212 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3213 #ifdef ASSERT 3214 // For now, the operand is an image. This will change once we start handling 3215 // more resource types. 3216 Register ResourcePtr = I.getOperand(2).getReg(); 3217 SPIRVType *RegType = GR.getResultType(ResourcePtr); 3218 assert(RegType->getOpcode() == SPIRV::OpTypeImage && 3219 "Can only handle texel buffers for now."); 3220 #endif 3221 3222 // For texel buffers, the index into the image is part of the OpImageRead or 3223 // OpImageWrite instructions. So we will do nothing in this case. This 3224 // intrinsic will be combined with the load or store when selecting the load 3225 // or store. 3226 return true; 3227 } 3228 3229 bool SPIRVInstructionSelector::extractSubvector( 3230 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3231 MachineInstr &InsertionPoint) const { 3232 SPIRVType *InputType = GR.getResultType(ReadReg); 3233 [[maybe_unused]] uint64_t InputSize = 3234 GR.getScalarOrVectorComponentCount(InputType); 3235 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3236 assert(InputSize > 1 && "The input must be a vector."); 3237 assert(ResultSize > 1 && "The result must be a vector."); 3238 assert(ResultSize < InputSize && 3239 "Cannot extract more element than there are in the input."); 3240 SmallVector<Register> ComponentRegisters; 3241 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3242 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3243 for (uint64_t I = 0; I < ResultSize; I++) { 3244 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3245 bool Succeed = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3246 InsertionPoint.getDebugLoc(), 3247 TII.get(SPIRV::OpCompositeExtract)) 3248 .addDef(ComponentReg) 3249 .addUse(ScalarType->getOperand(0).getReg()) 3250 .addUse(ReadReg) 3251 .addImm(I) 3252 .constrainAllUses(TII, TRI, RBI); 3253 if (!Succeed) 3254 return false; 3255 ComponentRegisters.emplace_back(ComponentReg); 3256 } 3257 3258 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3259 InsertionPoint.getDebugLoc(), 3260 TII.get(SPIRV::OpCompositeConstruct)) 3261 .addDef(ResVReg) 3262 .addUse(GR.getSPIRVTypeID(ResType)); 3263 3264 for (Register ComponentReg : ComponentRegisters) 3265 MIB.addUse(ComponentReg); 3266 return MIB.constrainAllUses(TII, TRI, RBI); 3267 } 3268 3269 bool SPIRVInstructionSelector::selectImageWriteIntrinsic( 3270 MachineInstr &I) const { 3271 // If the load of the image is in a different basic block, then 3272 // this will generate invalid code. A proper solution is to move 3273 // the OpLoad from selectHandleFromBinding here. However, to do 3274 // that we will need to change the return type of the intrinsic. 3275 // We will do that when we can, but for now trying to move forward with other 3276 // issues. 3277 Register ImageReg = I.getOperand(1).getReg(); 3278 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 3279 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 3280 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 3281 *ImageDef, I)) { 3282 return false; 3283 } 3284 3285 Register CoordinateReg = I.getOperand(2).getReg(); 3286 Register DataReg = I.getOperand(3).getReg(); 3287 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3288 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3289 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3290 TII.get(SPIRV::OpImageWrite)) 3291 .addUse(NewImageReg) 3292 .addUse(CoordinateReg) 3293 .addUse(DataReg) 3294 .constrainAllUses(TII, TRI, RBI); 3295 } 3296 3297 Register SPIRVInstructionSelector::buildPointerToResource( 3298 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3299 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3300 MachineIRBuilder MIRBuilder) const { 3301 if (ArraySize == 1) 3302 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3303 MIRBuilder); 3304 3305 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3306 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3307 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3308 VarType, Set, Binding, MIRBuilder); 3309 3310 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3311 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3312 3313 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3314 if (IsNonUniform) { 3315 // It is unclear which value needs to be marked an non-uniform, so both 3316 // the index and the access changed are decorated as non-uniform. 3317 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3318 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3319 } 3320 3321 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3322 .addDef(AcReg) 3323 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3324 .addUse(VarReg) 3325 .addUse(IndexReg); 3326 3327 return AcReg; 3328 } 3329 3330 bool SPIRVInstructionSelector::selectFirstBitSet16( 3331 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3332 unsigned ExtendOpcode, unsigned BitSetOpcode) const { 3333 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3334 bool Result = selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, 3335 ExtendOpcode); 3336 3337 return Result && 3338 selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode); 3339 } 3340 3341 bool SPIRVInstructionSelector::selectFirstBitSet32( 3342 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3343 Register SrcReg, unsigned BitSetOpcode) const { 3344 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3345 .addDef(ResVReg) 3346 .addUse(GR.getSPIRVTypeID(ResType)) 3347 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3348 .addImm(BitSetOpcode) 3349 .addUse(SrcReg) 3350 .constrainAllUses(TII, TRI, RBI); 3351 } 3352 3353 bool SPIRVInstructionSelector::selectFirstBitSet64Overflow( 3354 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3355 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3356 3357 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors 3358 // requires creating a param register and return register with an invalid 3359 // vector size. If that is resolved, then this function can be used for 3360 // vectors of any component size. 3361 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3362 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops"); 3363 3364 MachineIRBuilder MIRBuilder(I); 3365 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3366 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder); 3367 SPIRVType *I64x2Type = GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder); 3368 SPIRVType *Vec2ResType = 3369 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder); 3370 3371 std::vector<Register> PartialRegs; 3372 3373 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd 3374 unsigned CurrentComponent = 0; 3375 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) { 3376 // This register holds the firstbitX result for each of the i64x2 vectors 3377 // extracted from SrcReg 3378 Register BitSetResult = 3379 MRI->createVirtualRegister(GR.getRegClass(I64x2Type)); 3380 3381 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3382 TII.get(SPIRV::OpVectorShuffle)) 3383 .addDef(BitSetResult) 3384 .addUse(GR.getSPIRVTypeID(I64x2Type)) 3385 .addUse(SrcReg) 3386 .addUse(SrcReg) 3387 .addImm(CurrentComponent) 3388 .addImm(CurrentComponent + 1); 3389 3390 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3391 return false; 3392 3393 Register SubVecBitSetReg = 3394 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType)); 3395 3396 if (!selectFirstBitSet64(SubVecBitSetReg, Vec2ResType, I, BitSetResult, 3397 BitSetOpcode, SwapPrimarySide)) 3398 return false; 3399 3400 PartialRegs.push_back(SubVecBitSetReg); 3401 } 3402 3403 // On odd component counts we need to handle one more component 3404 if (CurrentComponent != ComponentCount) { 3405 bool ZeroAsNull = STI.isOpenCLEnv(); 3406 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type)); 3407 Register ConstIntLastIdx = GR.getOrCreateConstInt( 3408 ComponentCount - 1, I, BaseType, TII, ZeroAsNull); 3409 3410 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx}, 3411 SPIRV::OpVectorExtractDynamic)) 3412 return false; 3413 3414 Register FinalElemBitSetReg = 3415 MRI->createVirtualRegister(GR.getRegClass(BaseType)); 3416 3417 if (!selectFirstBitSet64(FinalElemBitSetReg, BaseType, I, FinalElemReg, 3418 BitSetOpcode, SwapPrimarySide)) 3419 return false; 3420 3421 PartialRegs.push_back(FinalElemBitSetReg); 3422 } 3423 3424 // Join all the resulting registers back into the return type in order 3425 // (ie i32x2, i32x2, i32x1 -> i32x5) 3426 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs, 3427 SPIRV::OpCompositeConstruct); 3428 } 3429 3430 bool SPIRVInstructionSelector::selectFirstBitSet64( 3431 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3432 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3433 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3434 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3435 bool ZeroAsNull = STI.isOpenCLEnv(); 3436 Register ConstIntZero = 3437 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull); 3438 Register ConstIntOne = 3439 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull); 3440 3441 // SPIRV doesn't support vectors with more than 4 components. Since the 3442 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only 3443 // operate on vectors with 2 or less components. When largers vectors are 3444 // seen. Split them, recurse, then recombine them. 3445 if (ComponentCount > 2) { 3446 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg, 3447 BitSetOpcode, SwapPrimarySide); 3448 } 3449 3450 // 1. Split int64 into 2 pieces using a bitcast 3451 MachineIRBuilder MIRBuilder(I); 3452 SPIRVType *PostCastType = 3453 GR.getOrCreateSPIRVVectorType(BaseType, 2 * ComponentCount, MIRBuilder); 3454 Register BitcastReg = 3455 MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3456 3457 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg}, 3458 SPIRV::OpBitcast)) 3459 return false; 3460 3461 // 2. Find the first set bit from the primary side for all the pieces in #1 3462 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3463 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode)) 3464 return false; 3465 3466 // 3. Split result vector into high bits and low bits 3467 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3468 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3469 3470 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3471 if (IsScalarRes) { 3472 // if scalar do a vector extract 3473 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntZero}, 3474 SPIRV::OpVectorExtractDynamic)) 3475 return false; 3476 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntOne}, 3477 SPIRV::OpVectorExtractDynamic)) 3478 return false; 3479 } else { 3480 // if vector do a shufflevector 3481 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3482 TII.get(SPIRV::OpVectorShuffle)) 3483 .addDef(HighReg) 3484 .addUse(GR.getSPIRVTypeID(ResType)) 3485 .addUse(FBSReg) 3486 // Per the spec, repeat the vector if only one vec is needed 3487 .addUse(FBSReg); 3488 3489 // high bits are stored in even indexes. Extract them from FBSReg 3490 for (unsigned J = 0; J < ComponentCount * 2; J += 2) { 3491 MIB.addImm(J); 3492 } 3493 3494 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3495 return false; 3496 3497 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3498 TII.get(SPIRV::OpVectorShuffle)) 3499 .addDef(LowReg) 3500 .addUse(GR.getSPIRVTypeID(ResType)) 3501 .addUse(FBSReg) 3502 // Per the spec, repeat the vector if only one vec is needed 3503 .addUse(FBSReg); 3504 3505 // low bits are stored in odd indexes. Extract them from FBSReg 3506 for (unsigned J = 1; J < ComponentCount * 2; J += 2) { 3507 MIB.addImm(J); 3508 } 3509 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3510 return false; 3511 } 3512 3513 // 4. Check the result. When primary bits == -1 use secondary, otherwise use 3514 // primary 3515 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3516 Register NegOneReg; 3517 Register Reg0; 3518 Register Reg32; 3519 unsigned SelectOp; 3520 unsigned AddOp; 3521 3522 if (IsScalarRes) { 3523 NegOneReg = 3524 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3525 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3526 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3527 SelectOp = SPIRV::OpSelectSISCond; 3528 AddOp = SPIRV::OpIAddS; 3529 } else { 3530 BoolType = 3531 GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount, MIRBuilder); 3532 NegOneReg = 3533 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3534 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3535 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3536 SelectOp = SPIRV::OpSelectVIVCond; 3537 AddOp = SPIRV::OpIAddV; 3538 } 3539 3540 Register PrimaryReg = HighReg; 3541 Register SecondaryReg = LowReg; 3542 Register PrimaryShiftReg = Reg32; 3543 Register SecondaryShiftReg = Reg0; 3544 3545 // By default the emitted opcodes check for the set bit from the MSB side. 3546 // Setting SwapPrimarySide checks the set bit from the LSB side 3547 if (SwapPrimarySide) { 3548 PrimaryReg = LowReg; 3549 SecondaryReg = HighReg; 3550 PrimaryShiftReg = Reg0; 3551 SecondaryShiftReg = Reg32; 3552 } 3553 3554 // Check if the primary bits are == -1 3555 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3556 if (!selectOpWithSrcs(BReg, BoolType, I, {PrimaryReg, NegOneReg}, 3557 SPIRV::OpIEqual)) 3558 return false; 3559 3560 // Select secondary bits if true in BReg, otherwise primary bits 3561 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3562 if (!selectOpWithSrcs(TmpReg, ResType, I, {BReg, SecondaryReg, PrimaryReg}, 3563 SelectOp)) 3564 return false; 3565 3566 // 5. Add 32 when high bits are used, otherwise 0 for low bits 3567 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3568 if (!selectOpWithSrcs(ValReg, ResType, I, 3569 {BReg, SecondaryShiftReg, PrimaryShiftReg}, SelectOp)) 3570 return false; 3571 3572 return selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, AddOp); 3573 } 3574 3575 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3576 const SPIRVType *ResType, 3577 MachineInstr &I, 3578 bool IsSigned) const { 3579 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3580 Register OpReg = I.getOperand(2).getReg(); 3581 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3582 // zero or sign extend 3583 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3584 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3585 3586 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3587 case 16: 3588 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3589 case 32: 3590 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3591 case 64: 3592 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3593 /*SwapPrimarySide=*/false); 3594 default: 3595 report_fatal_error( 3596 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3597 } 3598 } 3599 3600 bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg, 3601 const SPIRVType *ResType, 3602 MachineInstr &I) const { 3603 // FindILsb intrinsic only supports 32 bit integers 3604 Register OpReg = I.getOperand(2).getReg(); 3605 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3606 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it 3607 // to an unsigned i32. As this leaves all the least significant bits unchanged 3608 // so the first set bit from the LSB side doesn't change. 3609 unsigned ExtendOpcode = SPIRV::OpUConvert; 3610 unsigned BitSetOpcode = GL::FindILsb; 3611 3612 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3613 case 16: 3614 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3615 case 32: 3616 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3617 case 64: 3618 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3619 /*SwapPrimarySide=*/true); 3620 default: 3621 report_fatal_error("spv_firstbitlow only supports 16,32,64 bits."); 3622 } 3623 } 3624 3625 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3626 const SPIRVType *ResType, 3627 MachineInstr &I) const { 3628 // there was an allocation size parameter to the allocation instruction 3629 // that is not 1 3630 MachineBasicBlock &BB = *I.getParent(); 3631 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3632 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3633 .addDef(ResVReg) 3634 .addUse(GR.getSPIRVTypeID(ResType)) 3635 .addUse(I.getOperand(2).getReg()) 3636 .constrainAllUses(TII, TRI, RBI); 3637 if (!STI.isVulkanEnv()) { 3638 unsigned Alignment = I.getOperand(3).getImm(); 3639 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3640 } 3641 return Res; 3642 } 3643 3644 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3645 const SPIRVType *ResType, 3646 MachineInstr &I) const { 3647 // Change order of instructions if needed: all OpVariable instructions in a 3648 // function must be the first instructions in the first block 3649 auto It = getOpVariableMBBIt(I); 3650 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3651 TII.get(SPIRV::OpVariable)) 3652 .addDef(ResVReg) 3653 .addUse(GR.getSPIRVTypeID(ResType)) 3654 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3655 .constrainAllUses(TII, TRI, RBI); 3656 if (!STI.isVulkanEnv()) { 3657 unsigned Alignment = I.getOperand(2).getImm(); 3658 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3659 {Alignment}); 3660 } 3661 return Res; 3662 } 3663 3664 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3665 // InstructionSelector walks backwards through the instructions. We can use 3666 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3667 // first, so can generate an OpBranchConditional here. If there is no 3668 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3669 const MachineInstr *PrevI = I.getPrevNode(); 3670 MachineBasicBlock &MBB = *I.getParent(); 3671 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3672 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3673 .addUse(PrevI->getOperand(0).getReg()) 3674 .addMBB(PrevI->getOperand(1).getMBB()) 3675 .addMBB(I.getOperand(0).getMBB()) 3676 .constrainAllUses(TII, TRI, RBI); 3677 } 3678 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3679 .addMBB(I.getOperand(0).getMBB()) 3680 .constrainAllUses(TII, TRI, RBI); 3681 } 3682 3683 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3684 // InstructionSelector walks backwards through the instructions. For an 3685 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3686 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3687 // generate the OpBranchConditional in selectBranch above. 3688 // 3689 // If an OpBranchConditional has been generated, we simply return, as the work 3690 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3691 // implicit fallthrough to the next basic block, so we need to create an 3692 // OpBranchConditional with an explicit "false" argument pointing to the next 3693 // basic block that LLVM would fall through to. 3694 const MachineInstr *NextI = I.getNextNode(); 3695 // Check if this has already been successfully selected. 3696 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3697 return true; 3698 // Must be relying on implicit block fallthrough, so generate an 3699 // OpBranchConditional with the "next" basic block as the "false" target. 3700 MachineBasicBlock &MBB = *I.getParent(); 3701 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3702 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3703 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3704 .addUse(I.getOperand(0).getReg()) 3705 .addMBB(I.getOperand(1).getMBB()) 3706 .addMBB(NextMBB) 3707 .constrainAllUses(TII, TRI, RBI); 3708 } 3709 3710 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3711 const SPIRVType *ResType, 3712 MachineInstr &I) const { 3713 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3714 .addDef(ResVReg) 3715 .addUse(GR.getSPIRVTypeID(ResType)); 3716 const unsigned NumOps = I.getNumOperands(); 3717 for (unsigned i = 1; i < NumOps; i += 2) { 3718 MIB.addUse(I.getOperand(i + 0).getReg()); 3719 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3720 } 3721 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3722 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3723 MIB->removeOperand(1); 3724 return Res; 3725 } 3726 3727 bool SPIRVInstructionSelector::selectGlobalValue( 3728 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3729 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3730 MachineIRBuilder MIRBuilder(I); 3731 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3732 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3733 SPIRVType *PointerBaseType; 3734 if (GVType->isArrayTy()) { 3735 SPIRVType *ArrayElementType = 3736 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3737 SPIRV::AccessQualifier::ReadWrite, false); 3738 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3739 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3740 } else { 3741 PointerBaseType = GR.getOrCreateSPIRVType( 3742 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3743 } 3744 3745 std::string GlobalIdent; 3746 if (!GV->hasName()) { 3747 unsigned &ID = UnnamedGlobalIDs[GV]; 3748 if (ID == 0) 3749 ID = UnnamedGlobalIDs.size(); 3750 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3751 } else { 3752 GlobalIdent = GV->getName(); 3753 } 3754 3755 // Behaviour of functions as operands depends on availability of the 3756 // corresponding extension (SPV_INTEL_function_pointers): 3757 // - If there is an extension to operate with functions as operands: 3758 // We create a proper constant operand and evaluate a correct type for a 3759 // function pointer. 3760 // - Without the required extension: 3761 // We have functions as operands in tests with blocks of instruction e.g. in 3762 // transcoding/global_block.ll. These operands are not used and should be 3763 // substituted by zero constants. Their type is expected to be always 3764 // OpTypePointer Function %uchar. 3765 if (isa<Function>(GV)) { 3766 const Constant *ConstVal = GV; 3767 MachineBasicBlock &BB = *I.getParent(); 3768 Register NewReg = GR.find(ConstVal, GR.CurMF); 3769 if (!NewReg.isValid()) { 3770 Register NewReg = ResVReg; 3771 GR.add(ConstVal, GR.CurMF, NewReg); 3772 const Function *GVFun = 3773 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3774 ? dyn_cast<Function>(GV) 3775 : nullptr; 3776 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3777 PointerBaseType, I, TII, 3778 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3779 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3780 if (GVFun) { 3781 // References to a function via function pointers generate virtual 3782 // registers without a definition. We will resolve it later, during 3783 // module analysis stage. 3784 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 3785 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3786 Register FuncVReg = 3787 MRI->createGenericVirtualRegister(GR.getRegType(ResType)); 3788 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); 3789 MachineInstrBuilder MIB1 = 3790 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 3791 .addDef(FuncVReg) 3792 .addUse(ResTypeReg); 3793 MachineInstrBuilder MIB2 = 3794 BuildMI(BB, I, I.getDebugLoc(), 3795 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3796 .addDef(NewReg) 3797 .addUse(ResTypeReg) 3798 .addUse(FuncVReg); 3799 // mapping the function pointer to the used Function 3800 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); 3801 return MIB1.constrainAllUses(TII, TRI, RBI) && 3802 MIB2.constrainAllUses(TII, TRI, RBI); 3803 } 3804 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3805 .addDef(NewReg) 3806 .addUse(GR.getSPIRVTypeID(ResType)) 3807 .constrainAllUses(TII, TRI, RBI); 3808 } 3809 assert(NewReg != ResVReg); 3810 return BuildCOPY(ResVReg, NewReg, I); 3811 } 3812 auto GlobalVar = cast<GlobalVariable>(GV); 3813 assert(GlobalVar->getName() != "llvm.global.annotations"); 3814 3815 // Skip empty declaration for GVs with initializers till we get the decl with 3816 // passed initializer. 3817 if (hasInitializer(GlobalVar) && !Init) 3818 return true; 3819 3820 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage(); 3821 SPIRV::LinkageType::LinkageType LnkType = 3822 GV->isDeclarationForLinker() 3823 ? SPIRV::LinkageType::Import 3824 : (GV->hasLinkOnceODRLinkage() && 3825 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3826 ? SPIRV::LinkageType::LinkOnceODR 3827 : SPIRV::LinkageType::Export); 3828 3829 const unsigned AddrSpace = GV->getAddressSpace(); 3830 SPIRV::StorageClass::StorageClass StorageClass = 3831 addressSpaceToStorageClass(AddrSpace, STI); 3832 SPIRVType *ResType = 3833 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3834 Register Reg = GR.buildGlobalVariable( 3835 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3836 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3837 return Reg.isValid(); 3838 } 3839 3840 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3841 const SPIRVType *ResType, 3842 MachineInstr &I) const { 3843 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3844 return selectExtInst(ResVReg, ResType, I, CL::log10); 3845 } 3846 3847 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3848 // is implemented as: 3849 // log10(x) = log2(x) * (1 / log2(10)) 3850 // = log2(x) * 0.30103 3851 3852 MachineIRBuilder MIRBuilder(I); 3853 MachineBasicBlock &BB = *I.getParent(); 3854 3855 // Build log2(x). 3856 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3857 bool Result = 3858 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3859 .addDef(VarReg) 3860 .addUse(GR.getSPIRVTypeID(ResType)) 3861 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3862 .addImm(GL::Log2) 3863 .add(I.getOperand(1)) 3864 .constrainAllUses(TII, TRI, RBI); 3865 3866 // Build 0.30103. 3867 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3868 ResType->getOpcode() == SPIRV::OpTypeFloat); 3869 // TODO: Add matrix implementation once supported by the HLSL frontend. 3870 const SPIRVType *SpirvScalarType = 3871 ResType->getOpcode() == SPIRV::OpTypeVector 3872 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3873 : ResType; 3874 Register ScaleReg = 3875 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3876 3877 // Multiply log2(x) by 0.30103 to get log10(x) result. 3878 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3879 ? SPIRV::OpVectorTimesScalar 3880 : SPIRV::OpFMulS; 3881 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3882 .addDef(ResVReg) 3883 .addUse(GR.getSPIRVTypeID(ResType)) 3884 .addUse(VarReg) 3885 .addUse(ScaleReg) 3886 .constrainAllUses(TII, TRI, RBI); 3887 } 3888 3889 // Generate the instructions to load 3-element vector builtin input 3890 // IDs/Indices. 3891 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3892 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3893 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3894 const SPIRVType *ResType, MachineInstr &I) const { 3895 MachineIRBuilder MIRBuilder(I); 3896 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3897 const SPIRVType *Vec3Ty = 3898 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3899 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3900 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3901 3902 // Create new register for the input ID builtin variable. 3903 Register NewRegister = 3904 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3905 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3906 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3907 3908 // Build global variable with the necessary decorations for the input ID 3909 // builtin variable. 3910 Register Variable = GR.buildGlobalVariable( 3911 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3912 SPIRV::StorageClass::Input, nullptr, true, true, 3913 SPIRV::LinkageType::Import, MIRBuilder, false); 3914 3915 // Create new register for loading value. 3916 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3917 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3918 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3919 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3920 3921 // Load v3uint value from the global variable. 3922 bool Result = 3923 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3924 .addDef(LoadedRegister) 3925 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3926 .addUse(Variable); 3927 3928 // Get the input ID index. Expecting operand is a constant immediate value, 3929 // wrapped in a type assignment. 3930 assert(I.getOperand(2).isReg()); 3931 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3932 3933 // Extract the input ID from the loaded vector value. 3934 MachineBasicBlock &BB = *I.getParent(); 3935 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3936 .addDef(ResVReg) 3937 .addUse(GR.getSPIRVTypeID(ResType)) 3938 .addUse(LoadedRegister) 3939 .addImm(ThreadId); 3940 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3941 } 3942 3943 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3944 MachineInstr &I) const { 3945 MachineIRBuilder MIRBuilder(I); 3946 if (Type->getOpcode() != SPIRV::OpTypeVector) 3947 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3948 3949 uint64_t VectorSize = Type->getOperand(2).getImm(); 3950 if (VectorSize == 4) 3951 return Type; 3952 3953 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3954 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3955 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3956 } 3957 3958 bool SPIRVInstructionSelector::loadHandleBeforePosition( 3959 Register &HandleReg, const SPIRVType *ResType, GIntrinsic &HandleDef, 3960 MachineInstr &Pos) const { 3961 3962 assert(HandleDef.getIntrinsicID() == 3963 Intrinsic::spv_resource_handlefrombinding); 3964 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI); 3965 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI); 3966 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI); 3967 Register IndexReg = HandleDef.getOperand(5).getReg(); 3968 bool IsNonUniform = ArraySize > 1 && foldImm(HandleDef.getOperand(6), MRI); 3969 3970 MachineIRBuilder MIRBuilder(HandleDef); 3971 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3972 IndexReg, IsNonUniform, MIRBuilder); 3973 3974 if (IsNonUniform) 3975 buildOpDecorate(HandleReg, HandleDef, TII, SPIRV::Decoration::NonUniformEXT, 3976 {}); 3977 3978 // TODO: For now we assume the resource is an image, which needs to be 3979 // loaded to get the handle. That will not be true for storage buffers. 3980 return BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), 3981 TII.get(SPIRV::OpLoad)) 3982 .addDef(HandleReg) 3983 .addUse(GR.getSPIRVTypeID(ResType)) 3984 .addUse(VarReg) 3985 .constrainAllUses(TII, TRI, RBI); 3986 } 3987 3988 namespace llvm { 3989 InstructionSelector * 3990 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3991 const SPIRVSubtarget &Subtarget, 3992 const RegisterBankInfo &RBI) { 3993 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3994 } 3995 } // namespace llvm 3996