1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 #include "llvm/Support/ErrorHandling.h" 37 38 #define DEBUG_TYPE "spirv-isel" 39 40 using namespace llvm; 41 namespace CL = SPIRV::OpenCLExtInst; 42 namespace GL = SPIRV::GLSLExtInst; 43 44 using ExtInstList = 45 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 46 47 namespace { 48 49 llvm::SPIRV::SelectionControl::SelectionControl 50 getSelectionOperandForImm(int Imm) { 51 if (Imm == 2) 52 return SPIRV::SelectionControl::Flatten; 53 if (Imm == 1) 54 return SPIRV::SelectionControl::DontFlatten; 55 if (Imm == 0) 56 return SPIRV::SelectionControl::None; 57 llvm_unreachable("Invalid immediate"); 58 } 59 60 #define GET_GLOBALISEL_PREDICATE_BITSET 61 #include "SPIRVGenGlobalISel.inc" 62 #undef GET_GLOBALISEL_PREDICATE_BITSET 63 64 class SPIRVInstructionSelector : public InstructionSelector { 65 const SPIRVSubtarget &STI; 66 const SPIRVInstrInfo &TII; 67 const SPIRVRegisterInfo &TRI; 68 const RegisterBankInfo &RBI; 69 SPIRVGlobalRegistry &GR; 70 MachineRegisterInfo *MRI; 71 MachineFunction *HasVRegsReset = nullptr; 72 73 /// We need to keep track of the number we give to anonymous global values to 74 /// generate the same name every time when this is needed. 75 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 76 SmallPtrSet<MachineInstr *, 8> DeadMIs; 77 78 public: 79 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 80 const SPIRVSubtarget &ST, 81 const RegisterBankInfo &RBI); 82 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 83 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 84 BlockFrequencyInfo *BFI) override; 85 // Common selection code. Instruction-specific selection occurs in spvSelect. 86 bool select(MachineInstr &I) override; 87 static const char *getName() { return DEBUG_TYPE; } 88 89 #define GET_GLOBALISEL_PREDICATES_DECL 90 #include "SPIRVGenGlobalISel.inc" 91 #undef GET_GLOBALISEL_PREDICATES_DECL 92 93 #define GET_GLOBALISEL_TEMPORARIES_DECL 94 #include "SPIRVGenGlobalISel.inc" 95 #undef GET_GLOBALISEL_TEMPORARIES_DECL 96 97 private: 98 void resetVRegsType(MachineFunction &MF); 99 100 // tblgen-erated 'select' implementation, used as the initial selector for 101 // the patterns that don't require complex C++. 102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 103 104 // All instruction-specific selection that didn't happen in "select()". 105 // Is basically a large Switch/Case delegating to all other select method. 106 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I) const; 108 109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 110 MachineInstr &I, bool IsSigned) const; 111 112 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I) const; 114 115 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, unsigned ExtendOpcode, 117 unsigned BitSetOpcode) const; 118 119 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I, Register SrcReg, 121 unsigned BitSetOpcode) const; 122 123 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I, Register SrcReg, 125 unsigned BitSetOpcode, bool SwapPrimarySide) const; 126 127 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I, Register SrcReg, 129 unsigned BitSetOpcode, 130 bool SwapPrimarySide) const; 131 132 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 133 const MachineInstr *Init = nullptr) const; 134 135 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I, std::vector<Register> SrcRegs, 137 unsigned Opcode) const; 138 139 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 140 unsigned Opcode) const; 141 142 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 143 MachineInstr &I) const; 144 145 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 bool selectStore(MachineInstr &I) const; 148 149 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 bool selectStackRestore(MachineInstr &I) const; 152 153 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 154 155 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I, unsigned NewOpcode, 157 unsigned NegateOpcode = 0) const; 158 159 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 162 bool selectFence(MachineInstr &I) const; 163 164 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I, unsigned OpType) const; 169 170 bool selectAll(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectAny(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I) const; 181 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 185 unsigned comparisonOpcode, MachineInstr &I) const; 186 bool selectCross(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 192 MachineInstr &I) const; 193 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 194 MachineInstr &I) const; 195 196 bool selectSign(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 199 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 200 MachineInstr &I) const; 201 202 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I, unsigned Opcode) const; 204 205 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I, bool Signed) const; 207 208 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 209 MachineInstr &I) const; 210 211 template <bool Signed> 212 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 template <bool Signed> 215 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 216 MachineInstr &I) const; 217 218 bool selectWaveReduceMax(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I, bool IsUnsigned) const; 220 221 bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I) const; 223 224 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 225 int OpIdx) const; 226 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 227 int OpIdx) const; 228 229 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 230 MachineInstr &I) const; 231 232 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 233 bool IsSigned) const; 234 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 235 bool IsSigned, unsigned Opcode) const; 236 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 237 bool IsSigned) const; 238 239 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 240 MachineInstr &I) const; 241 242 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 243 bool IsSigned) const; 244 245 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 246 const SPIRVType *intTy, const SPIRVType *boolTy) const; 247 248 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 249 MachineInstr &I) const; 250 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 251 MachineInstr &I) const; 252 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 253 MachineInstr &I) const; 254 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I) const; 256 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 257 MachineInstr &I) const; 258 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 259 MachineInstr &I) const; 260 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 261 MachineInstr &I) const; 262 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 263 MachineInstr &I) const; 264 265 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 266 MachineInstr &I) const; 267 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 268 MachineInstr &I) const; 269 270 bool selectBranch(MachineInstr &I) const; 271 bool selectBranchCond(MachineInstr &I) const; 272 273 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 274 MachineInstr &I) const; 275 276 [[maybe_unused]] bool selectExtInst(Register ResVReg, 277 const SPIRVType *RestType, 278 MachineInstr &I, 279 GL::GLSLExtInst GLInst) const; 280 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 281 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 282 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 283 MachineInstr &I, CL::OpenCLExtInst CLInst, 284 GL::GLSLExtInst GLInst) const; 285 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 286 MachineInstr &I, const ExtInstList &ExtInsts) const; 287 288 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 289 MachineInstr &I) const; 290 291 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 292 MachineInstr &I) const; 293 294 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 295 MachineInstr &I, unsigned Opcode) const; 296 297 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 298 MachineInstr &I) const; 299 300 bool selectUnmergeValues(MachineInstr &I) const; 301 302 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 303 MachineInstr &I) const; 304 305 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 306 MachineInstr &I) const; 307 bool selectImageWriteIntrinsic(MachineInstr &I) const; 308 bool selectResourceGetPointer(Register &ResVReg, const SPIRVType *ResType, 309 MachineInstr &I) const; 310 311 // Utilities 312 std::pair<Register, bool> 313 buildI32Constant(uint32_t Val, MachineInstr &I, 314 const SPIRVType *ResType = nullptr) const; 315 316 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 317 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 318 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 319 MachineInstr &I) const; 320 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 321 322 bool wrapIntoSpecConstantOp(MachineInstr &I, 323 SmallVector<Register> &CompositeArgs) const; 324 325 Register getUcharPtrTypeReg(MachineInstr &I, 326 SPIRV::StorageClass::StorageClass SC) const; 327 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 328 Register Src, Register DestType, 329 uint32_t Opcode) const; 330 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 331 SPIRVType *SrcPtrTy) const; 332 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 333 uint32_t Binding, uint32_t ArraySize, 334 Register IndexReg, bool IsNonUniform, 335 MachineIRBuilder MIRBuilder) const; 336 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 337 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType, 338 Register &ReadReg, MachineInstr &InsertionPoint) const; 339 bool generateImageRead(Register &ResVReg, const SPIRVType *ResType, 340 Register ImageReg, Register IdxReg, DebugLoc Loc, 341 MachineInstr &Pos) const; 342 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 343 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 344 Register ResVReg, const SPIRVType *ResType, 345 MachineInstr &I) const; 346 bool loadHandleBeforePosition(Register &HandleReg, const SPIRVType *ResType, 347 GIntrinsic &HandleDef, MachineInstr &Pos) const; 348 }; 349 350 } // end anonymous namespace 351 352 #define GET_GLOBALISEL_IMPL 353 #include "SPIRVGenGlobalISel.inc" 354 #undef GET_GLOBALISEL_IMPL 355 356 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 357 const SPIRVSubtarget &ST, 358 const RegisterBankInfo &RBI) 359 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 360 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 361 #define GET_GLOBALISEL_PREDICATES_INIT 362 #include "SPIRVGenGlobalISel.inc" 363 #undef GET_GLOBALISEL_PREDICATES_INIT 364 #define GET_GLOBALISEL_TEMPORARIES_INIT 365 #include "SPIRVGenGlobalISel.inc" 366 #undef GET_GLOBALISEL_TEMPORARIES_INIT 367 { 368 } 369 370 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 371 CodeGenCoverage *CoverageInfo, 372 ProfileSummaryInfo *PSI, 373 BlockFrequencyInfo *BFI) { 374 MRI = &MF.getRegInfo(); 375 GR.setCurrentFunc(MF); 376 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 377 } 378 379 // Ensure that register classes correspond to pattern matching rules. 380 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 381 if (HasVRegsReset == &MF) 382 return; 383 HasVRegsReset = &MF; 384 385 MachineRegisterInfo &MRI = MF.getRegInfo(); 386 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 387 Register Reg = Register::index2VirtReg(I); 388 LLT RegType = MRI.getType(Reg); 389 if (RegType.isScalar()) 390 MRI.setType(Reg, LLT::scalar(64)); 391 else if (RegType.isPointer()) 392 MRI.setType(Reg, LLT::pointer(0, 64)); 393 else if (RegType.isVector()) 394 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 395 } 396 for (const auto &MBB : MF) { 397 for (const auto &MI : MBB) { 398 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 399 continue; 400 Register DstReg = MI.getOperand(0).getReg(); 401 LLT DstType = MRI.getType(DstReg); 402 Register SrcReg = MI.getOperand(1).getReg(); 403 LLT SrcType = MRI.getType(SrcReg); 404 if (DstType != SrcType) 405 MRI.setType(DstReg, MRI.getType(SrcReg)); 406 407 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 408 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 409 if (DstRC != SrcRC && SrcRC) 410 MRI.setRegClass(DstReg, SrcRC); 411 } 412 } 413 } 414 415 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 416 417 // Defined in SPIRVLegalizerInfo.cpp. 418 extern bool isTypeFoldingSupported(unsigned Opcode); 419 420 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 421 for (const auto &MO : MI.all_defs()) { 422 Register Reg = MO.getReg(); 423 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 424 return false; 425 } 426 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 427 MI.isLifetimeMarker()) 428 return false; 429 if (MI.isPHI()) 430 return true; 431 if (MI.mayStore() || MI.isCall() || 432 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 433 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 434 return false; 435 return true; 436 } 437 438 bool SPIRVInstructionSelector::select(MachineInstr &I) { 439 resetVRegsType(*I.getParent()->getParent()); 440 441 assert(I.getParent() && "Instruction should be in a basic block!"); 442 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 443 444 Register Opcode = I.getOpcode(); 445 // If it's not a GMIR instruction, we've selected it already. 446 if (!isPreISelGenericOpcode(Opcode)) { 447 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 448 Register DstReg = I.getOperand(0).getReg(); 449 Register SrcReg = I.getOperand(1).getReg(); 450 auto *Def = MRI->getVRegDef(SrcReg); 451 if (isTypeFoldingSupported(Def->getOpcode())) { 452 bool Res = selectImpl(I, *CoverageInfo); 453 LLVM_DEBUG({ 454 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 455 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 456 I.print(dbgs()); 457 } 458 }); 459 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 460 if (Res) { 461 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 462 DeadMIs.insert(Def); 463 return Res; 464 } 465 } 466 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 467 MRI->replaceRegWith(SrcReg, DstReg); 468 GR.invalidateMachineInstr(&I); 469 I.removeFromParent(); 470 return true; 471 } else if (I.getNumDefs() == 1) { 472 // Make all vregs 64 bits (for SPIR-V IDs). 473 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 474 } 475 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 476 } 477 478 if (DeadMIs.contains(&I)) { 479 // if the instruction has been already made dead by folding it away 480 // erase it 481 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 482 salvageDebugInfo(*MRI, I); 483 GR.invalidateMachineInstr(&I); 484 I.eraseFromParent(); 485 return true; 486 } 487 488 if (I.getNumOperands() != I.getNumExplicitOperands()) { 489 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 490 return false; 491 } 492 493 // Common code for getting return reg+type, and removing selected instr 494 // from parent occurs here. Instr-specific selection happens in spvSelect(). 495 bool HasDefs = I.getNumDefs() > 0; 496 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 497 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 498 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 499 if (spvSelect(ResVReg, ResType, I)) { 500 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 501 for (unsigned i = 0; i < I.getNumDefs(); ++i) 502 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 503 GR.invalidateMachineInstr(&I); 504 I.removeFromParent(); 505 return true; 506 } 507 return false; 508 } 509 510 static bool mayApplyGenericSelection(unsigned Opcode) { 511 switch (Opcode) { 512 case TargetOpcode::G_CONSTANT: 513 return false; 514 case TargetOpcode::G_SADDO: 515 case TargetOpcode::G_SSUBO: 516 return true; 517 } 518 return isTypeFoldingSupported(Opcode); 519 } 520 521 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 522 MachineInstr &I) const { 523 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 524 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 525 if (DstRC != SrcRC && SrcRC) 526 MRI->setRegClass(DestReg, SrcRC); 527 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 528 TII.get(TargetOpcode::COPY)) 529 .addDef(DestReg) 530 .addUse(SrcReg) 531 .constrainAllUses(TII, TRI, RBI); 532 } 533 534 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 535 const SPIRVType *ResType, 536 MachineInstr &I) const { 537 const unsigned Opcode = I.getOpcode(); 538 if (mayApplyGenericSelection(Opcode)) 539 return selectImpl(I, *CoverageInfo); 540 switch (Opcode) { 541 case TargetOpcode::G_CONSTANT: 542 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 543 I); 544 case TargetOpcode::G_GLOBAL_VALUE: 545 return selectGlobalValue(ResVReg, I); 546 case TargetOpcode::G_IMPLICIT_DEF: 547 return selectOpUndef(ResVReg, ResType, I); 548 case TargetOpcode::G_FREEZE: 549 return selectFreeze(ResVReg, ResType, I); 550 551 case TargetOpcode::G_INTRINSIC: 552 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 553 case TargetOpcode::G_INTRINSIC_CONVERGENT: 554 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 555 return selectIntrinsic(ResVReg, ResType, I); 556 case TargetOpcode::G_BITREVERSE: 557 return selectBitreverse(ResVReg, ResType, I); 558 559 case TargetOpcode::G_BUILD_VECTOR: 560 return selectBuildVector(ResVReg, ResType, I); 561 case TargetOpcode::G_SPLAT_VECTOR: 562 return selectSplatVector(ResVReg, ResType, I); 563 564 case TargetOpcode::G_SHUFFLE_VECTOR: { 565 MachineBasicBlock &BB = *I.getParent(); 566 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 567 .addDef(ResVReg) 568 .addUse(GR.getSPIRVTypeID(ResType)) 569 .addUse(I.getOperand(1).getReg()) 570 .addUse(I.getOperand(2).getReg()); 571 for (auto V : I.getOperand(3).getShuffleMask()) 572 MIB.addImm(V); 573 return MIB.constrainAllUses(TII, TRI, RBI); 574 } 575 case TargetOpcode::G_MEMMOVE: 576 case TargetOpcode::G_MEMCPY: 577 case TargetOpcode::G_MEMSET: 578 return selectMemOperation(ResVReg, I); 579 580 case TargetOpcode::G_ICMP: 581 return selectICmp(ResVReg, ResType, I); 582 case TargetOpcode::G_FCMP: 583 return selectFCmp(ResVReg, ResType, I); 584 585 case TargetOpcode::G_FRAME_INDEX: 586 return selectFrameIndex(ResVReg, ResType, I); 587 588 case TargetOpcode::G_LOAD: 589 return selectLoad(ResVReg, ResType, I); 590 case TargetOpcode::G_STORE: 591 return selectStore(I); 592 593 case TargetOpcode::G_BR: 594 return selectBranch(I); 595 case TargetOpcode::G_BRCOND: 596 return selectBranchCond(I); 597 598 case TargetOpcode::G_PHI: 599 return selectPhi(ResVReg, ResType, I); 600 601 case TargetOpcode::G_FPTOSI: 602 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 603 case TargetOpcode::G_FPTOUI: 604 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 605 606 case TargetOpcode::G_SITOFP: 607 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 608 case TargetOpcode::G_UITOFP: 609 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 610 611 case TargetOpcode::G_CTPOP: 612 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 613 case TargetOpcode::G_SMIN: 614 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 615 case TargetOpcode::G_UMIN: 616 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 617 618 case TargetOpcode::G_SMAX: 619 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 620 case TargetOpcode::G_UMAX: 621 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 622 623 case TargetOpcode::G_SCMP: 624 return selectSUCmp(ResVReg, ResType, I, true); 625 case TargetOpcode::G_UCMP: 626 return selectSUCmp(ResVReg, ResType, I, false); 627 628 case TargetOpcode::G_STRICT_FMA: 629 case TargetOpcode::G_FMA: 630 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 631 632 case TargetOpcode::G_STRICT_FLDEXP: 633 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 634 635 case TargetOpcode::G_FPOW: 636 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 637 case TargetOpcode::G_FPOWI: 638 return selectExtInst(ResVReg, ResType, I, CL::pown); 639 640 case TargetOpcode::G_FEXP: 641 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 642 case TargetOpcode::G_FEXP2: 643 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 644 645 case TargetOpcode::G_FLOG: 646 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 647 case TargetOpcode::G_FLOG2: 648 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 649 case TargetOpcode::G_FLOG10: 650 return selectLog10(ResVReg, ResType, I); 651 652 case TargetOpcode::G_FABS: 653 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 654 case TargetOpcode::G_ABS: 655 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 656 657 case TargetOpcode::G_FMINNUM: 658 case TargetOpcode::G_FMINIMUM: 659 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 660 case TargetOpcode::G_FMAXNUM: 661 case TargetOpcode::G_FMAXIMUM: 662 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 663 664 case TargetOpcode::G_FCOPYSIGN: 665 return selectExtInst(ResVReg, ResType, I, CL::copysign); 666 667 case TargetOpcode::G_FCEIL: 668 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 669 case TargetOpcode::G_FFLOOR: 670 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 671 672 case TargetOpcode::G_FCOS: 673 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 674 case TargetOpcode::G_FSIN: 675 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 676 case TargetOpcode::G_FTAN: 677 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 678 case TargetOpcode::G_FACOS: 679 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 680 case TargetOpcode::G_FASIN: 681 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 682 case TargetOpcode::G_FATAN: 683 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 684 case TargetOpcode::G_FATAN2: 685 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 686 case TargetOpcode::G_FCOSH: 687 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 688 case TargetOpcode::G_FSINH: 689 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 690 case TargetOpcode::G_FTANH: 691 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 692 693 case TargetOpcode::G_STRICT_FSQRT: 694 case TargetOpcode::G_FSQRT: 695 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 696 697 case TargetOpcode::G_CTTZ: 698 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 699 return selectExtInst(ResVReg, ResType, I, CL::ctz); 700 case TargetOpcode::G_CTLZ: 701 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 702 return selectExtInst(ResVReg, ResType, I, CL::clz); 703 704 case TargetOpcode::G_INTRINSIC_ROUND: 705 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 706 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 707 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 708 case TargetOpcode::G_INTRINSIC_TRUNC: 709 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 710 case TargetOpcode::G_FRINT: 711 case TargetOpcode::G_FNEARBYINT: 712 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 713 714 case TargetOpcode::G_SMULH: 715 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 716 case TargetOpcode::G_UMULH: 717 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 718 719 case TargetOpcode::G_SADDSAT: 720 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 721 case TargetOpcode::G_UADDSAT: 722 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 723 case TargetOpcode::G_SSUBSAT: 724 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 725 case TargetOpcode::G_USUBSAT: 726 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 727 728 case TargetOpcode::G_UADDO: 729 return selectOverflowArith(ResVReg, ResType, I, 730 ResType->getOpcode() == SPIRV::OpTypeVector 731 ? SPIRV::OpIAddCarryV 732 : SPIRV::OpIAddCarryS); 733 case TargetOpcode::G_USUBO: 734 return selectOverflowArith(ResVReg, ResType, I, 735 ResType->getOpcode() == SPIRV::OpTypeVector 736 ? SPIRV::OpISubBorrowV 737 : SPIRV::OpISubBorrowS); 738 case TargetOpcode::G_UMULO: 739 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 740 case TargetOpcode::G_SMULO: 741 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 742 743 case TargetOpcode::G_SEXT: 744 return selectExt(ResVReg, ResType, I, true); 745 case TargetOpcode::G_ANYEXT: 746 case TargetOpcode::G_ZEXT: 747 return selectExt(ResVReg, ResType, I, false); 748 case TargetOpcode::G_TRUNC: 749 return selectTrunc(ResVReg, ResType, I); 750 case TargetOpcode::G_FPTRUNC: 751 case TargetOpcode::G_FPEXT: 752 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 753 754 case TargetOpcode::G_PTRTOINT: 755 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 756 case TargetOpcode::G_INTTOPTR: 757 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 758 case TargetOpcode::G_BITCAST: 759 return selectBitcast(ResVReg, ResType, I); 760 case TargetOpcode::G_ADDRSPACE_CAST: 761 return selectAddrSpaceCast(ResVReg, ResType, I); 762 case TargetOpcode::G_PTR_ADD: { 763 // Currently, we get G_PTR_ADD only applied to global variables. 764 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 765 Register GV = I.getOperand(1).getReg(); 766 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 767 (void)II; 768 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 769 (*II).getOpcode() == TargetOpcode::COPY || 770 (*II).getOpcode() == SPIRV::OpVariable) && 771 isImm(I.getOperand(2), MRI)); 772 // It may be the initialization of a global variable. 773 bool IsGVInit = false; 774 for (MachineRegisterInfo::use_instr_iterator 775 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 776 UseEnd = MRI->use_instr_end(); 777 UseIt != UseEnd; UseIt = std::next(UseIt)) { 778 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 779 (*UseIt).getOpcode() == SPIRV::OpVariable) { 780 IsGVInit = true; 781 break; 782 } 783 } 784 MachineBasicBlock &BB = *I.getParent(); 785 if (!IsGVInit) { 786 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 787 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 788 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 789 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 790 // Build a new virtual register that is associated with the required 791 // data type. 792 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 793 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 794 // Having a correctly typed base we are ready to build the actually 795 // required GEP. It may not be a constant though, because all Operands 796 // of OpSpecConstantOp is to originate from other const instructions, 797 // and only the AccessChain named opcodes accept a global OpVariable 798 // instruction. We can't use an AccessChain opcode because of the type 799 // mismatch between result and base types. 800 if (!GR.isBitcastCompatible(ResType, GVType)) 801 report_fatal_error( 802 "incompatible result and operand types in a bitcast"); 803 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 804 MachineInstrBuilder MIB = 805 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 806 .addDef(NewVReg) 807 .addUse(ResTypeReg) 808 .addUse(GV); 809 return MIB.constrainAllUses(TII, TRI, RBI) && 810 BuildMI(BB, I, I.getDebugLoc(), 811 TII.get(STI.isVulkanEnv() 812 ? SPIRV::OpInBoundsAccessChain 813 : SPIRV::OpInBoundsPtrAccessChain)) 814 .addDef(ResVReg) 815 .addUse(ResTypeReg) 816 .addUse(NewVReg) 817 .addUse(I.getOperand(2).getReg()) 818 .constrainAllUses(TII, TRI, RBI); 819 } else { 820 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 821 .addDef(ResVReg) 822 .addUse(GR.getSPIRVTypeID(ResType)) 823 .addImm( 824 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 825 .addUse(GV) 826 .addUse(I.getOperand(2).getReg()) 827 .constrainAllUses(TII, TRI, RBI); 828 } 829 } 830 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 831 // initialize a global variable with a constant expression (e.g., the test 832 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 833 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 834 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 835 .addDef(ResVReg) 836 .addUse(GR.getSPIRVTypeID(ResType)) 837 .addImm(static_cast<uint32_t>( 838 SPIRV::Opcode::InBoundsPtrAccessChain)) 839 .addUse(GV) 840 .addUse(Idx) 841 .addUse(I.getOperand(2).getReg()); 842 return MIB.constrainAllUses(TII, TRI, RBI); 843 } 844 845 case TargetOpcode::G_ATOMICRMW_OR: 846 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 847 case TargetOpcode::G_ATOMICRMW_ADD: 848 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 849 case TargetOpcode::G_ATOMICRMW_AND: 850 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 851 case TargetOpcode::G_ATOMICRMW_MAX: 852 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 853 case TargetOpcode::G_ATOMICRMW_MIN: 854 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 855 case TargetOpcode::G_ATOMICRMW_SUB: 856 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 857 case TargetOpcode::G_ATOMICRMW_XOR: 858 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 859 case TargetOpcode::G_ATOMICRMW_UMAX: 860 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 861 case TargetOpcode::G_ATOMICRMW_UMIN: 862 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 863 case TargetOpcode::G_ATOMICRMW_XCHG: 864 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 865 case TargetOpcode::G_ATOMIC_CMPXCHG: 866 return selectAtomicCmpXchg(ResVReg, ResType, I); 867 868 case TargetOpcode::G_ATOMICRMW_FADD: 869 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 870 case TargetOpcode::G_ATOMICRMW_FSUB: 871 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 872 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 873 SPIRV::OpFNegate); 874 case TargetOpcode::G_ATOMICRMW_FMIN: 875 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 876 case TargetOpcode::G_ATOMICRMW_FMAX: 877 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 878 879 case TargetOpcode::G_FENCE: 880 return selectFence(I); 881 882 case TargetOpcode::G_STACKSAVE: 883 return selectStackSave(ResVReg, ResType, I); 884 case TargetOpcode::G_STACKRESTORE: 885 return selectStackRestore(I); 886 887 case TargetOpcode::G_UNMERGE_VALUES: 888 return selectUnmergeValues(I); 889 890 // Discard gen opcodes for intrinsics which we do not expect to actually 891 // represent code after lowering or intrinsics which are not implemented but 892 // should not crash when found in a customer's LLVM IR input. 893 case TargetOpcode::G_TRAP: 894 case TargetOpcode::G_DEBUGTRAP: 895 case TargetOpcode::G_UBSANTRAP: 896 case TargetOpcode::DBG_LABEL: 897 return true; 898 899 default: 900 return false; 901 } 902 } 903 904 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 905 const SPIRVType *ResType, 906 MachineInstr &I, 907 GL::GLSLExtInst GLInst) const { 908 return selectExtInst(ResVReg, ResType, I, 909 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 910 } 911 912 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 913 const SPIRVType *ResType, 914 MachineInstr &I, 915 CL::OpenCLExtInst CLInst) const { 916 return selectExtInst(ResVReg, ResType, I, 917 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 918 } 919 920 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 921 const SPIRVType *ResType, 922 MachineInstr &I, 923 CL::OpenCLExtInst CLInst, 924 GL::GLSLExtInst GLInst) const { 925 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 926 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 927 return selectExtInst(ResVReg, ResType, I, ExtInsts); 928 } 929 930 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 931 const SPIRVType *ResType, 932 MachineInstr &I, 933 const ExtInstList &Insts) const { 934 935 for (const auto &Ex : Insts) { 936 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 937 uint32_t Opcode = Ex.second; 938 if (STI.canUseExtInstSet(Set)) { 939 MachineBasicBlock &BB = *I.getParent(); 940 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 941 .addDef(ResVReg) 942 .addUse(GR.getSPIRVTypeID(ResType)) 943 .addImm(static_cast<uint32_t>(Set)) 944 .addImm(Opcode); 945 const unsigned NumOps = I.getNumOperands(); 946 unsigned Index = 1; 947 if (Index < NumOps && 948 I.getOperand(Index).getType() == 949 MachineOperand::MachineOperandType::MO_IntrinsicID) 950 Index = 2; 951 for (; Index < NumOps; ++Index) 952 MIB.add(I.getOperand(Index)); 953 return MIB.constrainAllUses(TII, TRI, RBI); 954 } 955 } 956 return false; 957 } 958 959 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 960 const SPIRVType *ResType, 961 MachineInstr &I, 962 std::vector<Register> Srcs, 963 unsigned Opcode) const { 964 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 965 .addDef(ResVReg) 966 .addUse(GR.getSPIRVTypeID(ResType)); 967 for (Register SReg : Srcs) { 968 MIB.addUse(SReg); 969 } 970 return MIB.constrainAllUses(TII, TRI, RBI); 971 } 972 973 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 974 const SPIRVType *ResType, 975 MachineInstr &I, 976 unsigned Opcode) const { 977 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 978 Register SrcReg = I.getOperand(1).getReg(); 979 bool IsGV = false; 980 for (MachineRegisterInfo::def_instr_iterator DefIt = 981 MRI->def_instr_begin(SrcReg); 982 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 983 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 984 IsGV = true; 985 break; 986 } 987 } 988 if (IsGV) { 989 uint32_t SpecOpcode = 0; 990 switch (Opcode) { 991 case SPIRV::OpConvertPtrToU: 992 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 993 break; 994 case SPIRV::OpConvertUToPtr: 995 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 996 break; 997 } 998 if (SpecOpcode) 999 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1000 TII.get(SPIRV::OpSpecConstantOp)) 1001 .addDef(ResVReg) 1002 .addUse(GR.getSPIRVTypeID(ResType)) 1003 .addImm(SpecOpcode) 1004 .addUse(SrcReg) 1005 .constrainAllUses(TII, TRI, RBI); 1006 } 1007 } 1008 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 1009 Opcode); 1010 } 1011 1012 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 1013 const SPIRVType *ResType, 1014 MachineInstr &I) const { 1015 Register OpReg = I.getOperand(1).getReg(); 1016 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 1017 if (!GR.isBitcastCompatible(ResType, OpType)) 1018 report_fatal_error("incompatible result and operand types in a bitcast"); 1019 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1020 } 1021 1022 static void addMemoryOperands(MachineMemOperand *MemOp, 1023 MachineInstrBuilder &MIB) { 1024 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1025 if (MemOp->isVolatile()) 1026 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1027 if (MemOp->isNonTemporal()) 1028 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1029 if (MemOp->getAlign().value()) 1030 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 1031 1032 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 1033 MIB.addImm(SpvMemOp); 1034 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 1035 MIB.addImm(MemOp->getAlign().value()); 1036 } 1037 } 1038 1039 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1040 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1041 if (Flags & MachineMemOperand::Flags::MOVolatile) 1042 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1043 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1044 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1045 1046 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1047 MIB.addImm(SpvMemOp); 1048 } 1049 1050 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1051 const SPIRVType *ResType, 1052 MachineInstr &I) const { 1053 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1054 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1055 1056 auto *PtrDef = getVRegDef(*MRI, Ptr); 1057 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); 1058 if (IntPtrDef && 1059 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 1060 Register ImageReg = IntPtrDef->getOperand(2).getReg(); 1061 Register NewImageReg = 1062 MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 1063 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 1064 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 1065 *ImageDef, I)) { 1066 return false; 1067 } 1068 1069 Register IdxReg = IntPtrDef->getOperand(3).getReg(); 1070 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, 1071 I.getDebugLoc(), I); 1072 } 1073 1074 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1075 .addDef(ResVReg) 1076 .addUse(GR.getSPIRVTypeID(ResType)) 1077 .addUse(Ptr); 1078 if (!I.getNumMemOperands()) { 1079 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1080 I.getOpcode() == 1081 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1082 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1083 } else { 1084 addMemoryOperands(*I.memoperands_begin(), MIB); 1085 } 1086 return MIB.constrainAllUses(TII, TRI, RBI); 1087 } 1088 1089 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1090 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1091 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1092 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1093 1094 auto *PtrDef = getVRegDef(*MRI, Ptr); 1095 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); 1096 if (IntPtrDef && 1097 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 1098 Register ImageReg = IntPtrDef->getOperand(2).getReg(); 1099 Register NewImageReg = 1100 MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 1101 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 1102 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 1103 *ImageDef, I)) { 1104 return false; 1105 } 1106 1107 Register IdxReg = IntPtrDef->getOperand(3).getReg(); 1108 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1109 TII.get(SPIRV::OpImageWrite)) 1110 .addUse(NewImageReg) 1111 .addUse(IdxReg) 1112 .addUse(StoreVal) 1113 .constrainAllUses(TII, TRI, RBI); 1114 } 1115 1116 MachineBasicBlock &BB = *I.getParent(); 1117 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1118 .addUse(Ptr) 1119 .addUse(StoreVal); 1120 if (!I.getNumMemOperands()) { 1121 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1122 I.getOpcode() == 1123 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1124 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1125 } else { 1126 addMemoryOperands(*I.memoperands_begin(), MIB); 1127 } 1128 return MIB.constrainAllUses(TII, TRI, RBI); 1129 } 1130 1131 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1132 const SPIRVType *ResType, 1133 MachineInstr &I) const { 1134 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1135 report_fatal_error( 1136 "llvm.stacksave intrinsic: this instruction requires the following " 1137 "SPIR-V extension: SPV_INTEL_variable_length_array", 1138 false); 1139 MachineBasicBlock &BB = *I.getParent(); 1140 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1141 .addDef(ResVReg) 1142 .addUse(GR.getSPIRVTypeID(ResType)) 1143 .constrainAllUses(TII, TRI, RBI); 1144 } 1145 1146 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1147 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1148 report_fatal_error( 1149 "llvm.stackrestore intrinsic: this instruction requires the following " 1150 "SPIR-V extension: SPV_INTEL_variable_length_array", 1151 false); 1152 if (!I.getOperand(0).isReg()) 1153 return false; 1154 MachineBasicBlock &BB = *I.getParent(); 1155 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1156 .addUse(I.getOperand(0).getReg()) 1157 .constrainAllUses(TII, TRI, RBI); 1158 } 1159 1160 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1161 MachineInstr &I) const { 1162 MachineBasicBlock &BB = *I.getParent(); 1163 Register SrcReg = I.getOperand(1).getReg(); 1164 bool Result = true; 1165 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1166 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1167 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1168 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1169 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1170 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1171 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1172 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1173 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1174 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1175 Function &CurFunction = GR.CurMF->getFunction(); 1176 Type *LLVMArrTy = 1177 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1178 // Module takes ownership of the global var. 1179 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1180 true, GlobalValue::InternalLinkage, 1181 Constant::getNullValue(LLVMArrTy)); 1182 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1183 GR.add(GV, GR.CurMF, VarReg); 1184 GR.addGlobalObject(GV, GR.CurMF, VarReg); 1185 1186 Result &= 1187 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1188 .addDef(VarReg) 1189 .addUse(GR.getSPIRVTypeID(VarTy)) 1190 .addImm(SPIRV::StorageClass::UniformConstant) 1191 .addUse(Const) 1192 .constrainAllUses(TII, TRI, RBI); 1193 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1194 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1195 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1196 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1197 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1198 } 1199 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1200 .addUse(I.getOperand(0).getReg()) 1201 .addUse(SrcReg) 1202 .addUse(I.getOperand(2).getReg()); 1203 if (I.getNumMemOperands()) 1204 addMemoryOperands(*I.memoperands_begin(), MIB); 1205 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1206 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1207 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1208 return Result; 1209 } 1210 1211 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1212 const SPIRVType *ResType, 1213 MachineInstr &I, 1214 unsigned NewOpcode, 1215 unsigned NegateOpcode) const { 1216 bool Result = true; 1217 assert(I.hasOneMemOperand()); 1218 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1219 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1220 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1221 auto ScopeConstant = buildI32Constant(Scope, I); 1222 Register ScopeReg = ScopeConstant.first; 1223 Result &= ScopeConstant.second; 1224 1225 Register Ptr = I.getOperand(1).getReg(); 1226 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1227 // auto ScSem = 1228 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1229 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1230 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1231 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1232 Register MemSemReg = MemSemConstant.first; 1233 Result &= MemSemConstant.second; 1234 1235 Register ValueReg = I.getOperand(2).getReg(); 1236 if (NegateOpcode != 0) { 1237 // Translation with negative value operand is requested 1238 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1239 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1240 ValueReg = TmpReg; 1241 } 1242 1243 return Result && 1244 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1245 .addDef(ResVReg) 1246 .addUse(GR.getSPIRVTypeID(ResType)) 1247 .addUse(Ptr) 1248 .addUse(ScopeReg) 1249 .addUse(MemSemReg) 1250 .addUse(ValueReg) 1251 .constrainAllUses(TII, TRI, RBI); 1252 } 1253 1254 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1255 unsigned ArgI = I.getNumOperands() - 1; 1256 Register SrcReg = 1257 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1258 SPIRVType *DefType = 1259 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1260 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1261 report_fatal_error( 1262 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1263 1264 SPIRVType *ScalarType = 1265 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1266 MachineBasicBlock &BB = *I.getParent(); 1267 bool Res = false; 1268 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1269 Register ResVReg = I.getOperand(i).getReg(); 1270 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1271 if (!ResType) { 1272 // There was no "assign type" actions, let's fix this now 1273 ResType = ScalarType; 1274 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1275 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1276 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1277 } 1278 auto MIB = 1279 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1280 .addDef(ResVReg) 1281 .addUse(GR.getSPIRVTypeID(ResType)) 1282 .addUse(SrcReg) 1283 .addImm(static_cast<int64_t>(i)); 1284 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1285 } 1286 return Res; 1287 } 1288 1289 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1290 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1291 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1292 auto MemSemConstant = buildI32Constant(MemSem, I); 1293 Register MemSemReg = MemSemConstant.first; 1294 bool Result = MemSemConstant.second; 1295 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1296 uint32_t Scope = static_cast<uint32_t>( 1297 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1298 auto ScopeConstant = buildI32Constant(Scope, I); 1299 Register ScopeReg = ScopeConstant.first; 1300 Result &= ScopeConstant.second; 1301 MachineBasicBlock &BB = *I.getParent(); 1302 return Result && 1303 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1304 .addUse(ScopeReg) 1305 .addUse(MemSemReg) 1306 .constrainAllUses(TII, TRI, RBI); 1307 } 1308 1309 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1310 const SPIRVType *ResType, 1311 MachineInstr &I, 1312 unsigned Opcode) const { 1313 Type *ResTy = nullptr; 1314 StringRef ResName; 1315 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1316 report_fatal_error( 1317 "Not enough info to select the arithmetic with overflow instruction"); 1318 if (!ResTy || !ResTy->isStructTy()) 1319 report_fatal_error("Expect struct type result for the arithmetic " 1320 "with overflow instruction"); 1321 // "Result Type must be from OpTypeStruct. The struct must have two members, 1322 // and the two members must be the same type." 1323 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1324 ResTy = StructType::get(ResElemTy, ResElemTy); 1325 // Build SPIR-V types and constant(s) if needed. 1326 MachineIRBuilder MIRBuilder(I); 1327 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1328 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1329 assert(I.getNumDefs() > 1 && "Not enought operands"); 1330 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1331 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1332 if (N > 1) 1333 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1334 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1335 Register ZeroReg = buildZerosVal(ResType, I); 1336 // A new virtual register to store the result struct. 1337 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1338 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1339 // Build the result name if needed. 1340 if (ResName.size() > 0) 1341 buildOpName(StructVReg, ResName, MIRBuilder); 1342 // Build the arithmetic with overflow instruction. 1343 MachineBasicBlock &BB = *I.getParent(); 1344 auto MIB = 1345 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1346 .addDef(StructVReg) 1347 .addUse(GR.getSPIRVTypeID(StructType)); 1348 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1349 MIB.addUse(I.getOperand(i).getReg()); 1350 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1351 // Build instructions to extract fields of the instruction's result. 1352 // A new virtual register to store the higher part of the result struct. 1353 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1354 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1355 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1356 auto MIB = 1357 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1358 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1359 .addUse(GR.getSPIRVTypeID(ResType)) 1360 .addUse(StructVReg) 1361 .addImm(i); 1362 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1363 } 1364 // Build boolean value from the higher part. 1365 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1366 .addDef(I.getOperand(1).getReg()) 1367 .addUse(BoolTypeReg) 1368 .addUse(HigherVReg) 1369 .addUse(ZeroReg) 1370 .constrainAllUses(TII, TRI, RBI); 1371 } 1372 1373 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1374 const SPIRVType *ResType, 1375 MachineInstr &I) const { 1376 bool Result = true; 1377 Register ScopeReg; 1378 Register MemSemEqReg; 1379 Register MemSemNeqReg; 1380 Register Ptr = I.getOperand(2).getReg(); 1381 if (!isa<GIntrinsic>(I)) { 1382 assert(I.hasOneMemOperand()); 1383 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1384 unsigned Scope = static_cast<uint32_t>(getMemScope( 1385 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1386 auto ScopeConstant = buildI32Constant(Scope, I); 1387 ScopeReg = ScopeConstant.first; 1388 Result &= ScopeConstant.second; 1389 1390 unsigned ScSem = static_cast<uint32_t>( 1391 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1392 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1393 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1394 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1395 MemSemEqReg = MemSemEqConstant.first; 1396 Result &= MemSemEqConstant.second; 1397 AtomicOrdering FO = MemOp->getFailureOrdering(); 1398 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1399 if (MemSemEq == MemSemNeq) 1400 MemSemNeqReg = MemSemEqReg; 1401 else { 1402 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1403 MemSemNeqReg = MemSemNeqConstant.first; 1404 Result &= MemSemNeqConstant.second; 1405 } 1406 } else { 1407 ScopeReg = I.getOperand(5).getReg(); 1408 MemSemEqReg = I.getOperand(6).getReg(); 1409 MemSemNeqReg = I.getOperand(7).getReg(); 1410 } 1411 1412 Register Cmp = I.getOperand(3).getReg(); 1413 Register Val = I.getOperand(4).getReg(); 1414 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1415 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1416 const DebugLoc &DL = I.getDebugLoc(); 1417 Result &= 1418 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1419 .addDef(ACmpRes) 1420 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1421 .addUse(Ptr) 1422 .addUse(ScopeReg) 1423 .addUse(MemSemEqReg) 1424 .addUse(MemSemNeqReg) 1425 .addUse(Val) 1426 .addUse(Cmp) 1427 .constrainAllUses(TII, TRI, RBI); 1428 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1429 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1430 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1431 .addDef(CmpSuccReg) 1432 .addUse(GR.getSPIRVTypeID(BoolTy)) 1433 .addUse(ACmpRes) 1434 .addUse(Cmp) 1435 .constrainAllUses(TII, TRI, RBI); 1436 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1437 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1438 .addDef(TmpReg) 1439 .addUse(GR.getSPIRVTypeID(ResType)) 1440 .addUse(ACmpRes) 1441 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1442 .addImm(0) 1443 .constrainAllUses(TII, TRI, RBI); 1444 return Result && 1445 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1446 .addDef(ResVReg) 1447 .addUse(GR.getSPIRVTypeID(ResType)) 1448 .addUse(CmpSuccReg) 1449 .addUse(TmpReg) 1450 .addImm(1) 1451 .constrainAllUses(TII, TRI, RBI); 1452 } 1453 1454 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1455 switch (SC) { 1456 case SPIRV::StorageClass::Workgroup: 1457 case SPIRV::StorageClass::CrossWorkgroup: 1458 case SPIRV::StorageClass::Function: 1459 return true; 1460 default: 1461 return false; 1462 } 1463 } 1464 1465 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1466 switch (SC) { 1467 case SPIRV::StorageClass::DeviceOnlyINTEL: 1468 case SPIRV::StorageClass::HostOnlyINTEL: 1469 return true; 1470 default: 1471 return false; 1472 } 1473 } 1474 1475 // Returns true ResVReg is referred only from global vars and OpName's. 1476 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1477 bool IsGRef = false; 1478 bool IsAllowedRefs = 1479 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1480 [&IsGRef](auto const &It) { 1481 unsigned Opcode = It.getOpcode(); 1482 if (Opcode == SPIRV::OpConstantComposite || 1483 Opcode == SPIRV::OpVariable || 1484 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1485 return IsGRef = true; 1486 return Opcode == SPIRV::OpName; 1487 }); 1488 return IsAllowedRefs && IsGRef; 1489 } 1490 1491 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1492 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1493 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1494 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1495 } 1496 1497 MachineInstrBuilder 1498 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1499 Register Src, Register DestType, 1500 uint32_t Opcode) const { 1501 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1502 TII.get(SPIRV::OpSpecConstantOp)) 1503 .addDef(Dest) 1504 .addUse(DestType) 1505 .addImm(Opcode) 1506 .addUse(Src); 1507 } 1508 1509 MachineInstrBuilder 1510 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1511 SPIRVType *SrcPtrTy) const { 1512 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1513 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1514 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1515 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1516 SPIRV::StorageClass::Generic), 1517 GR.getPointerSize())); 1518 MachineFunction *MF = I.getParent()->getParent(); 1519 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1520 MachineInstrBuilder MIB = buildSpecConstantOp( 1521 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1522 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1523 GR.add(MIB.getInstr(), MF, Tmp); 1524 return MIB; 1525 } 1526 1527 // In SPIR-V address space casting can only happen to and from the Generic 1528 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1529 // pointers to and from Generic pointers. As such, we can convert e.g. from 1530 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1531 // other combinations can only be done by a bitcast, and are probably not safe. 1532 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1533 const SPIRVType *ResType, 1534 MachineInstr &I) const { 1535 MachineBasicBlock &BB = *I.getParent(); 1536 const DebugLoc &DL = I.getDebugLoc(); 1537 1538 Register SrcPtr = I.getOperand(1).getReg(); 1539 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1540 1541 // don't generate a cast for a null that may be represented by OpTypeInt 1542 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1543 ResType->getOpcode() != SPIRV::OpTypePointer) 1544 return BuildCOPY(ResVReg, SrcPtr, I); 1545 1546 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1547 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1548 1549 if (isASCastInGVar(MRI, ResVReg)) { 1550 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1551 // are expressed by OpSpecConstantOp with an Opcode. 1552 // TODO: maybe insert a check whether the Kernel capability was declared and 1553 // so PtrCastToGeneric/GenericCastToPtr are available. 1554 unsigned SpecOpcode = 1555 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1556 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1557 : (SrcSC == SPIRV::StorageClass::Generic && 1558 isGenericCastablePtr(DstSC) 1559 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1560 : 0); 1561 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1562 // correct value of ResType and use general i8* instead. Maybe this should 1563 // be addressed in the emit-intrinsic step to infer a correct 1564 // OpConstantComposite type. 1565 if (SpecOpcode) { 1566 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1567 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1568 .constrainAllUses(TII, TRI, RBI); 1569 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1570 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1571 return MIB.constrainAllUses(TII, TRI, RBI) && 1572 buildSpecConstantOp( 1573 I, ResVReg, MIB->getOperand(0).getReg(), 1574 getUcharPtrTypeReg(I, DstSC), 1575 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1576 .constrainAllUses(TII, TRI, RBI); 1577 } 1578 } 1579 1580 // don't generate a cast between identical storage classes 1581 if (SrcSC == DstSC) 1582 return BuildCOPY(ResVReg, SrcPtr, I); 1583 1584 if ((SrcSC == SPIRV::StorageClass::Function && 1585 DstSC == SPIRV::StorageClass::Private) || 1586 (DstSC == SPIRV::StorageClass::Function && 1587 SrcSC == SPIRV::StorageClass::Private)) 1588 return BuildCOPY(ResVReg, SrcPtr, I); 1589 1590 // Casting from an eligible pointer to Generic. 1591 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1592 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1593 // Casting from Generic to an eligible pointer. 1594 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1595 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1596 // Casting between 2 eligible pointers using Generic as an intermediary. 1597 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1598 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1599 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1600 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1601 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1602 .addDef(Tmp) 1603 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1604 .addUse(SrcPtr) 1605 .constrainAllUses(TII, TRI, RBI); 1606 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1607 .addDef(ResVReg) 1608 .addUse(GR.getSPIRVTypeID(ResType)) 1609 .addUse(Tmp) 1610 .constrainAllUses(TII, TRI, RBI); 1611 } 1612 1613 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1614 // be applied 1615 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1616 return selectUnOp(ResVReg, ResType, I, 1617 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1618 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1619 return selectUnOp(ResVReg, ResType, I, 1620 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1621 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1622 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1623 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1624 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1625 1626 // Bitcast for pointers requires that the address spaces must match 1627 return false; 1628 } 1629 1630 static unsigned getFCmpOpcode(unsigned PredNum) { 1631 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1632 switch (Pred) { 1633 case CmpInst::FCMP_OEQ: 1634 return SPIRV::OpFOrdEqual; 1635 case CmpInst::FCMP_OGE: 1636 return SPIRV::OpFOrdGreaterThanEqual; 1637 case CmpInst::FCMP_OGT: 1638 return SPIRV::OpFOrdGreaterThan; 1639 case CmpInst::FCMP_OLE: 1640 return SPIRV::OpFOrdLessThanEqual; 1641 case CmpInst::FCMP_OLT: 1642 return SPIRV::OpFOrdLessThan; 1643 case CmpInst::FCMP_ONE: 1644 return SPIRV::OpFOrdNotEqual; 1645 case CmpInst::FCMP_ORD: 1646 return SPIRV::OpOrdered; 1647 case CmpInst::FCMP_UEQ: 1648 return SPIRV::OpFUnordEqual; 1649 case CmpInst::FCMP_UGE: 1650 return SPIRV::OpFUnordGreaterThanEqual; 1651 case CmpInst::FCMP_UGT: 1652 return SPIRV::OpFUnordGreaterThan; 1653 case CmpInst::FCMP_ULE: 1654 return SPIRV::OpFUnordLessThanEqual; 1655 case CmpInst::FCMP_ULT: 1656 return SPIRV::OpFUnordLessThan; 1657 case CmpInst::FCMP_UNE: 1658 return SPIRV::OpFUnordNotEqual; 1659 case CmpInst::FCMP_UNO: 1660 return SPIRV::OpUnordered; 1661 default: 1662 llvm_unreachable("Unknown predicate type for FCmp"); 1663 } 1664 } 1665 1666 static unsigned getICmpOpcode(unsigned PredNum) { 1667 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1668 switch (Pred) { 1669 case CmpInst::ICMP_EQ: 1670 return SPIRV::OpIEqual; 1671 case CmpInst::ICMP_NE: 1672 return SPIRV::OpINotEqual; 1673 case CmpInst::ICMP_SGE: 1674 return SPIRV::OpSGreaterThanEqual; 1675 case CmpInst::ICMP_SGT: 1676 return SPIRV::OpSGreaterThan; 1677 case CmpInst::ICMP_SLE: 1678 return SPIRV::OpSLessThanEqual; 1679 case CmpInst::ICMP_SLT: 1680 return SPIRV::OpSLessThan; 1681 case CmpInst::ICMP_UGE: 1682 return SPIRV::OpUGreaterThanEqual; 1683 case CmpInst::ICMP_UGT: 1684 return SPIRV::OpUGreaterThan; 1685 case CmpInst::ICMP_ULE: 1686 return SPIRV::OpULessThanEqual; 1687 case CmpInst::ICMP_ULT: 1688 return SPIRV::OpULessThan; 1689 default: 1690 llvm_unreachable("Unknown predicate type for ICmp"); 1691 } 1692 } 1693 1694 static unsigned getPtrCmpOpcode(unsigned Pred) { 1695 switch (static_cast<CmpInst::Predicate>(Pred)) { 1696 case CmpInst::ICMP_EQ: 1697 return SPIRV::OpPtrEqual; 1698 case CmpInst::ICMP_NE: 1699 return SPIRV::OpPtrNotEqual; 1700 default: 1701 llvm_unreachable("Unknown predicate type for pointer comparison"); 1702 } 1703 } 1704 1705 // Return the logical operation, or abort if none exists. 1706 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1707 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1708 switch (Pred) { 1709 case CmpInst::ICMP_EQ: 1710 return SPIRV::OpLogicalEqual; 1711 case CmpInst::ICMP_NE: 1712 return SPIRV::OpLogicalNotEqual; 1713 default: 1714 llvm_unreachable("Unknown predicate type for Bool comparison"); 1715 } 1716 } 1717 1718 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1719 if (!LLVMFloatTy) 1720 return APFloat::getZero(APFloat::IEEEsingle()); 1721 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1722 case Type::HalfTyID: 1723 return APFloat::getZero(APFloat::IEEEhalf()); 1724 default: 1725 case Type::FloatTyID: 1726 return APFloat::getZero(APFloat::IEEEsingle()); 1727 case Type::DoubleTyID: 1728 return APFloat::getZero(APFloat::IEEEdouble()); 1729 } 1730 } 1731 1732 static APFloat getOneFP(const Type *LLVMFloatTy) { 1733 if (!LLVMFloatTy) 1734 return APFloat::getOne(APFloat::IEEEsingle()); 1735 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1736 case Type::HalfTyID: 1737 return APFloat::getOne(APFloat::IEEEhalf()); 1738 default: 1739 case Type::FloatTyID: 1740 return APFloat::getOne(APFloat::IEEEsingle()); 1741 case Type::DoubleTyID: 1742 return APFloat::getOne(APFloat::IEEEdouble()); 1743 } 1744 } 1745 1746 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1747 const SPIRVType *ResType, 1748 MachineInstr &I, 1749 unsigned OpAnyOrAll) const { 1750 assert(I.getNumOperands() == 3); 1751 assert(I.getOperand(2).isReg()); 1752 MachineBasicBlock &BB = *I.getParent(); 1753 Register InputRegister = I.getOperand(2).getReg(); 1754 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1755 1756 if (!InputType) 1757 report_fatal_error("Input Type could not be determined."); 1758 1759 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1760 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1761 if (IsBoolTy && !IsVectorTy) { 1762 assert(ResVReg == I.getOperand(0).getReg()); 1763 return BuildCOPY(ResVReg, InputRegister, I); 1764 } 1765 1766 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1767 unsigned SpirvNotEqualId = 1768 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1769 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1770 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1771 Register NotEqualReg = ResVReg; 1772 1773 if (IsVectorTy) { 1774 NotEqualReg = IsBoolTy ? InputRegister 1775 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1776 const unsigned NumElts = InputType->getOperand(2).getImm(); 1777 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1778 } 1779 1780 bool Result = true; 1781 if (!IsBoolTy) { 1782 Register ConstZeroReg = 1783 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1784 1785 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1786 .addDef(NotEqualReg) 1787 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1788 .addUse(InputRegister) 1789 .addUse(ConstZeroReg) 1790 .constrainAllUses(TII, TRI, RBI); 1791 } 1792 1793 if (!IsVectorTy) 1794 return Result; 1795 1796 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1797 .addDef(ResVReg) 1798 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1799 .addUse(NotEqualReg) 1800 .constrainAllUses(TII, TRI, RBI); 1801 } 1802 1803 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1804 const SPIRVType *ResType, 1805 MachineInstr &I) const { 1806 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1807 } 1808 1809 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1810 const SPIRVType *ResType, 1811 MachineInstr &I) const { 1812 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1813 } 1814 1815 // Select the OpDot instruction for the given float dot 1816 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1817 const SPIRVType *ResType, 1818 MachineInstr &I) const { 1819 assert(I.getNumOperands() == 4); 1820 assert(I.getOperand(2).isReg()); 1821 assert(I.getOperand(3).isReg()); 1822 1823 [[maybe_unused]] SPIRVType *VecType = 1824 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1825 1826 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1827 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1828 "dot product requires a vector of at least 2 components"); 1829 1830 [[maybe_unused]] SPIRVType *EltType = 1831 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1832 1833 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1834 1835 MachineBasicBlock &BB = *I.getParent(); 1836 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1837 .addDef(ResVReg) 1838 .addUse(GR.getSPIRVTypeID(ResType)) 1839 .addUse(I.getOperand(2).getReg()) 1840 .addUse(I.getOperand(3).getReg()) 1841 .constrainAllUses(TII, TRI, RBI); 1842 } 1843 1844 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1845 const SPIRVType *ResType, 1846 MachineInstr &I, 1847 bool Signed) const { 1848 assert(I.getNumOperands() == 4); 1849 assert(I.getOperand(2).isReg()); 1850 assert(I.getOperand(3).isReg()); 1851 MachineBasicBlock &BB = *I.getParent(); 1852 1853 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1854 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1855 .addDef(ResVReg) 1856 .addUse(GR.getSPIRVTypeID(ResType)) 1857 .addUse(I.getOperand(2).getReg()) 1858 .addUse(I.getOperand(3).getReg()) 1859 .constrainAllUses(TII, TRI, RBI); 1860 } 1861 1862 // Since pre-1.6 SPIRV has no integer dot implementation, 1863 // expand by piecewise multiplying and adding the results 1864 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1865 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1866 assert(I.getNumOperands() == 4); 1867 assert(I.getOperand(2).isReg()); 1868 assert(I.getOperand(3).isReg()); 1869 MachineBasicBlock &BB = *I.getParent(); 1870 1871 // Multiply the vectors, then sum the results 1872 Register Vec0 = I.getOperand(2).getReg(); 1873 Register Vec1 = I.getOperand(3).getReg(); 1874 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1875 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1876 1877 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1878 .addDef(TmpVec) 1879 .addUse(GR.getSPIRVTypeID(VecType)) 1880 .addUse(Vec0) 1881 .addUse(Vec1) 1882 .constrainAllUses(TII, TRI, RBI); 1883 1884 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1885 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1886 "dot product requires a vector of at least 2 components"); 1887 1888 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1889 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1890 .addDef(Res) 1891 .addUse(GR.getSPIRVTypeID(ResType)) 1892 .addUse(TmpVec) 1893 .addImm(0) 1894 .constrainAllUses(TII, TRI, RBI); 1895 1896 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1897 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1898 1899 Result &= 1900 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1901 .addDef(Elt) 1902 .addUse(GR.getSPIRVTypeID(ResType)) 1903 .addUse(TmpVec) 1904 .addImm(i) 1905 .constrainAllUses(TII, TRI, RBI); 1906 1907 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1908 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1909 : ResVReg; 1910 1911 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1912 .addDef(Sum) 1913 .addUse(GR.getSPIRVTypeID(ResType)) 1914 .addUse(Res) 1915 .addUse(Elt) 1916 .constrainAllUses(TII, TRI, RBI); 1917 Res = Sum; 1918 } 1919 1920 return Result; 1921 } 1922 1923 template <bool Signed> 1924 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1925 const SPIRVType *ResType, 1926 MachineInstr &I) const { 1927 assert(I.getNumOperands() == 5); 1928 assert(I.getOperand(2).isReg()); 1929 assert(I.getOperand(3).isReg()); 1930 assert(I.getOperand(4).isReg()); 1931 MachineBasicBlock &BB = *I.getParent(); 1932 1933 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1934 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1935 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1936 .addDef(Dot) 1937 .addUse(GR.getSPIRVTypeID(ResType)) 1938 .addUse(I.getOperand(2).getReg()) 1939 .addUse(I.getOperand(3).getReg()) 1940 .constrainAllUses(TII, TRI, RBI); 1941 1942 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1943 .addDef(ResVReg) 1944 .addUse(GR.getSPIRVTypeID(ResType)) 1945 .addUse(Dot) 1946 .addUse(I.getOperand(4).getReg()) 1947 .constrainAllUses(TII, TRI, RBI); 1948 } 1949 1950 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1951 // extract the elements of the packed inputs, multiply them and add the result 1952 // to the accumulator. 1953 template <bool Signed> 1954 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1955 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1956 assert(I.getNumOperands() == 5); 1957 assert(I.getOperand(2).isReg()); 1958 assert(I.getOperand(3).isReg()); 1959 assert(I.getOperand(4).isReg()); 1960 MachineBasicBlock &BB = *I.getParent(); 1961 1962 bool Result = true; 1963 1964 // Acc = C 1965 Register Acc = I.getOperand(4).getReg(); 1966 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1967 auto ExtractOp = 1968 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1969 1970 // Extract the i8 element, multiply and add it to the accumulator 1971 for (unsigned i = 0; i < 4; i++) { 1972 // A[i] 1973 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1974 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1975 .addDef(AElt) 1976 .addUse(GR.getSPIRVTypeID(ResType)) 1977 .addUse(I.getOperand(2).getReg()) 1978 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1979 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1980 .constrainAllUses(TII, TRI, RBI); 1981 1982 // B[i] 1983 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1984 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1985 .addDef(BElt) 1986 .addUse(GR.getSPIRVTypeID(ResType)) 1987 .addUse(I.getOperand(3).getReg()) 1988 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1989 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1990 .constrainAllUses(TII, TRI, RBI); 1991 1992 // A[i] * B[i] 1993 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1994 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1995 .addDef(Mul) 1996 .addUse(GR.getSPIRVTypeID(ResType)) 1997 .addUse(AElt) 1998 .addUse(BElt) 1999 .constrainAllUses(TII, TRI, RBI); 2000 2001 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 2002 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2003 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 2004 .addDef(MaskMul) 2005 .addUse(GR.getSPIRVTypeID(ResType)) 2006 .addUse(Mul) 2007 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 2008 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 2009 .constrainAllUses(TII, TRI, RBI); 2010 2011 // Acc = Acc + A[i] * B[i] 2012 Register Sum = 2013 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 2014 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 2015 .addDef(Sum) 2016 .addUse(GR.getSPIRVTypeID(ResType)) 2017 .addUse(Acc) 2018 .addUse(MaskMul) 2019 .constrainAllUses(TII, TRI, RBI); 2020 2021 Acc = Sum; 2022 } 2023 2024 return Result; 2025 } 2026 2027 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 2028 /// does not have a saturate builtin. 2029 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 2030 const SPIRVType *ResType, 2031 MachineInstr &I) const { 2032 assert(I.getNumOperands() == 3); 2033 assert(I.getOperand(2).isReg()); 2034 MachineBasicBlock &BB = *I.getParent(); 2035 Register VZero = buildZerosValF(ResType, I); 2036 Register VOne = buildOnesValF(ResType, I); 2037 2038 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2039 .addDef(ResVReg) 2040 .addUse(GR.getSPIRVTypeID(ResType)) 2041 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2042 .addImm(GL::FClamp) 2043 .addUse(I.getOperand(2).getReg()) 2044 .addUse(VZero) 2045 .addUse(VOne) 2046 .constrainAllUses(TII, TRI, RBI); 2047 } 2048 2049 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 2050 const SPIRVType *ResType, 2051 MachineInstr &I) const { 2052 assert(I.getNumOperands() == 3); 2053 assert(I.getOperand(2).isReg()); 2054 MachineBasicBlock &BB = *I.getParent(); 2055 Register InputRegister = I.getOperand(2).getReg(); 2056 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2057 auto &DL = I.getDebugLoc(); 2058 2059 if (!InputType) 2060 report_fatal_error("Input Type could not be determined."); 2061 2062 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2063 2064 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 2065 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 2066 2067 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 2068 2069 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 2070 Register SignReg = NeedsConversion 2071 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 2072 : ResVReg; 2073 2074 bool Result = 2075 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 2076 .addDef(SignReg) 2077 .addUse(GR.getSPIRVTypeID(InputType)) 2078 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2079 .addImm(SignOpcode) 2080 .addUse(InputRegister) 2081 .constrainAllUses(TII, TRI, RBI); 2082 2083 if (NeedsConversion) { 2084 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2085 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2086 .addDef(ResVReg) 2087 .addUse(GR.getSPIRVTypeID(ResType)) 2088 .addUse(SignReg) 2089 .constrainAllUses(TII, TRI, RBI); 2090 } 2091 2092 return Result; 2093 } 2094 2095 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2096 const SPIRVType *ResType, 2097 MachineInstr &I, 2098 unsigned Opcode) const { 2099 MachineBasicBlock &BB = *I.getParent(); 2100 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2101 2102 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2103 .addDef(ResVReg) 2104 .addUse(GR.getSPIRVTypeID(ResType)) 2105 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2106 IntTy, TII)); 2107 2108 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2109 BMI.addUse(I.getOperand(J).getReg()); 2110 } 2111 2112 return BMI.constrainAllUses(TII, TRI, RBI); 2113 } 2114 2115 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2116 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2117 2118 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2119 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2120 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2121 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2122 SPIRV::OpGroupNonUniformBallot); 2123 2124 MachineBasicBlock &BB = *I.getParent(); 2125 Result &= 2126 BuildMI(BB, I, I.getDebugLoc(), 2127 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2128 .addDef(ResVReg) 2129 .addUse(GR.getSPIRVTypeID(ResType)) 2130 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2131 .addImm(SPIRV::GroupOperation::Reduce) 2132 .addUse(BallotReg) 2133 .constrainAllUses(TII, TRI, RBI); 2134 2135 return Result; 2136 } 2137 2138 bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg, 2139 const SPIRVType *ResType, 2140 MachineInstr &I, 2141 bool IsUnsigned) const { 2142 assert(I.getNumOperands() == 3); 2143 assert(I.getOperand(2).isReg()); 2144 MachineBasicBlock &BB = *I.getParent(); 2145 Register InputRegister = I.getOperand(2).getReg(); 2146 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2147 2148 if (!InputType) 2149 report_fatal_error("Input Type could not be determined."); 2150 2151 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2152 // Retreive the operation to use based on input type 2153 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2154 auto IntegerOpcodeType = 2155 IsUnsigned ? SPIRV::OpGroupNonUniformUMax : SPIRV::OpGroupNonUniformSMax; 2156 auto Opcode = IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntegerOpcodeType; 2157 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2158 .addDef(ResVReg) 2159 .addUse(GR.getSPIRVTypeID(ResType)) 2160 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2161 .addImm(SPIRV::GroupOperation::Reduce) 2162 .addUse(I.getOperand(2).getReg()) 2163 .constrainAllUses(TII, TRI, RBI); 2164 } 2165 2166 bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg, 2167 const SPIRVType *ResType, 2168 MachineInstr &I) const { 2169 assert(I.getNumOperands() == 3); 2170 assert(I.getOperand(2).isReg()); 2171 MachineBasicBlock &BB = *I.getParent(); 2172 Register InputRegister = I.getOperand(2).getReg(); 2173 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2174 2175 if (!InputType) 2176 report_fatal_error("Input Type could not be determined."); 2177 2178 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2179 // Retreive the operation to use based on input type 2180 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2181 auto Opcode = 2182 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd : SPIRV::OpGroupNonUniformIAdd; 2183 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2184 .addDef(ResVReg) 2185 .addUse(GR.getSPIRVTypeID(ResType)) 2186 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2187 .addImm(SPIRV::GroupOperation::Reduce) 2188 .addUse(I.getOperand(2).getReg()); 2189 } 2190 2191 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2192 const SPIRVType *ResType, 2193 MachineInstr &I) const { 2194 MachineBasicBlock &BB = *I.getParent(); 2195 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2196 .addDef(ResVReg) 2197 .addUse(GR.getSPIRVTypeID(ResType)) 2198 .addUse(I.getOperand(1).getReg()) 2199 .constrainAllUses(TII, TRI, RBI); 2200 } 2201 2202 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2203 const SPIRVType *ResType, 2204 MachineInstr &I) const { 2205 // There is no way to implement `freeze` correctly without support on SPIR-V 2206 // standard side, but we may at least address a simple (static) case when 2207 // undef/poison value presence is obvious. The main benefit of even 2208 // incomplete `freeze` support is preventing of translation from crashing due 2209 // to lack of support on legalization and instruction selection steps. 2210 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2211 return false; 2212 Register OpReg = I.getOperand(1).getReg(); 2213 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2214 Register Reg; 2215 switch (Def->getOpcode()) { 2216 case SPIRV::ASSIGN_TYPE: 2217 if (MachineInstr *AssignToDef = 2218 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2219 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2220 Reg = Def->getOperand(2).getReg(); 2221 } 2222 break; 2223 case SPIRV::OpUndef: 2224 Reg = Def->getOperand(1).getReg(); 2225 break; 2226 } 2227 unsigned DestOpCode; 2228 if (Reg.isValid()) { 2229 DestOpCode = SPIRV::OpConstantNull; 2230 } else { 2231 DestOpCode = TargetOpcode::COPY; 2232 Reg = OpReg; 2233 } 2234 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2235 .addDef(I.getOperand(0).getReg()) 2236 .addUse(Reg) 2237 .constrainAllUses(TII, TRI, RBI); 2238 } 2239 return false; 2240 } 2241 2242 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2243 const SPIRVType *ResType) { 2244 Register OpReg = ResType->getOperand(2).getReg(); 2245 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2246 if (!OpDef) 2247 return 0; 2248 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2249 OpDef->getOperand(1).isReg()) { 2250 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2251 OpDef = RefDef; 2252 } 2253 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2254 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2255 : 0; 2256 return N; 2257 } 2258 2259 // Return true if the type represents a constant register 2260 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2261 SmallPtrSet<SPIRVType *, 4> &Visited) { 2262 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2263 OpDef->getOperand(1).isReg()) { 2264 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2265 OpDef = RefDef; 2266 } 2267 2268 if (Visited.contains(OpDef)) 2269 return true; 2270 Visited.insert(OpDef); 2271 2272 unsigned Opcode = OpDef->getOpcode(); 2273 switch (Opcode) { 2274 case TargetOpcode::G_CONSTANT: 2275 case TargetOpcode::G_FCONSTANT: 2276 return true; 2277 case TargetOpcode::G_INTRINSIC: 2278 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2279 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2280 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2281 Intrinsic::spv_const_composite; 2282 case TargetOpcode::G_BUILD_VECTOR: 2283 case TargetOpcode::G_SPLAT_VECTOR: { 2284 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2285 i++) { 2286 SPIRVType *OpNestedDef = 2287 OpDef->getOperand(i).isReg() 2288 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2289 : nullptr; 2290 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2291 return false; 2292 } 2293 return true; 2294 } 2295 } 2296 return false; 2297 } 2298 2299 // Return true if the virtual register represents a constant 2300 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2301 SmallPtrSet<SPIRVType *, 4> Visited; 2302 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2303 return isConstReg(MRI, OpDef, Visited); 2304 return false; 2305 } 2306 2307 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2308 const SPIRVType *ResType, 2309 MachineInstr &I) const { 2310 unsigned N = 0; 2311 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2312 N = GR.getScalarOrVectorComponentCount(ResType); 2313 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2314 N = getArrayComponentCount(MRI, ResType); 2315 else 2316 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2317 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2318 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2319 2320 // check if we may construct a constant vector 2321 bool IsConst = true; 2322 for (unsigned i = I.getNumExplicitDefs(); 2323 i < I.getNumExplicitOperands() && IsConst; ++i) 2324 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2325 IsConst = false; 2326 2327 if (!IsConst && N < 2) 2328 report_fatal_error( 2329 "There must be at least two constituent operands in a vector"); 2330 2331 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2332 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2333 TII.get(IsConst ? SPIRV::OpConstantComposite 2334 : SPIRV::OpCompositeConstruct)) 2335 .addDef(ResVReg) 2336 .addUse(GR.getSPIRVTypeID(ResType)); 2337 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2338 MIB.addUse(I.getOperand(i).getReg()); 2339 return MIB.constrainAllUses(TII, TRI, RBI); 2340 } 2341 2342 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2343 const SPIRVType *ResType, 2344 MachineInstr &I) const { 2345 unsigned N = 0; 2346 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2347 N = GR.getScalarOrVectorComponentCount(ResType); 2348 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2349 N = getArrayComponentCount(MRI, ResType); 2350 else 2351 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2352 2353 unsigned OpIdx = I.getNumExplicitDefs(); 2354 if (!I.getOperand(OpIdx).isReg()) 2355 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2356 2357 // check if we may construct a constant vector 2358 Register OpReg = I.getOperand(OpIdx).getReg(); 2359 bool IsConst = isConstReg(MRI, OpReg); 2360 2361 if (!IsConst && N < 2) 2362 report_fatal_error( 2363 "There must be at least two constituent operands in a vector"); 2364 2365 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2366 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2367 TII.get(IsConst ? SPIRV::OpConstantComposite 2368 : SPIRV::OpCompositeConstruct)) 2369 .addDef(ResVReg) 2370 .addUse(GR.getSPIRVTypeID(ResType)); 2371 for (unsigned i = 0; i < N; ++i) 2372 MIB.addUse(OpReg); 2373 return MIB.constrainAllUses(TII, TRI, RBI); 2374 } 2375 2376 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2377 const SPIRVType *ResType, 2378 MachineInstr &I) const { 2379 2380 unsigned Opcode; 2381 2382 if (STI.canUseExtension( 2383 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2384 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2385 Opcode = SPIRV::OpDemoteToHelperInvocation; 2386 } else { 2387 Opcode = SPIRV::OpKill; 2388 // OpKill must be the last operation of any basic block. 2389 if (MachineInstr *NextI = I.getNextNode()) { 2390 GR.invalidateMachineInstr(NextI); 2391 NextI->removeFromParent(); 2392 } 2393 } 2394 2395 MachineBasicBlock &BB = *I.getParent(); 2396 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2397 .constrainAllUses(TII, TRI, RBI); 2398 } 2399 2400 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2401 const SPIRVType *ResType, 2402 unsigned CmpOpc, 2403 MachineInstr &I) const { 2404 Register Cmp0 = I.getOperand(2).getReg(); 2405 Register Cmp1 = I.getOperand(3).getReg(); 2406 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2407 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2408 "CMP operands should have the same type"); 2409 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2410 .addDef(ResVReg) 2411 .addUse(GR.getSPIRVTypeID(ResType)) 2412 .addUse(Cmp0) 2413 .addUse(Cmp1) 2414 .constrainAllUses(TII, TRI, RBI); 2415 } 2416 2417 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2418 const SPIRVType *ResType, 2419 MachineInstr &I) const { 2420 auto Pred = I.getOperand(1).getPredicate(); 2421 unsigned CmpOpc; 2422 2423 Register CmpOperand = I.getOperand(2).getReg(); 2424 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2425 CmpOpc = getPtrCmpOpcode(Pred); 2426 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2427 CmpOpc = getBoolCmpOpcode(Pred); 2428 else 2429 CmpOpc = getICmpOpcode(Pred); 2430 return selectCmp(ResVReg, ResType, CmpOpc, I); 2431 } 2432 2433 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2434 const MachineInstr &I, 2435 int OpIdx) const { 2436 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2437 "Expected G_FCONSTANT"); 2438 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2439 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2440 } 2441 2442 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2443 const MachineInstr &I, 2444 int OpIdx) const { 2445 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2446 "Expected G_CONSTANT"); 2447 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2448 } 2449 2450 std::pair<Register, bool> 2451 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2452 const SPIRVType *ResType) const { 2453 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2454 const SPIRVType *SpvI32Ty = 2455 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2456 // Find a constant in DT or build a new one. 2457 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2458 Register NewReg = GR.find(ConstInt, GR.CurMF); 2459 bool Result = true; 2460 if (!NewReg.isValid()) { 2461 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2462 GR.add(ConstInt, GR.CurMF, NewReg); 2463 MachineInstr *MI; 2464 MachineBasicBlock &BB = *I.getParent(); 2465 if (Val == 0) { 2466 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2467 .addDef(NewReg) 2468 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2469 } else { 2470 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2471 .addDef(NewReg) 2472 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2473 .addImm(APInt(32, Val).getZExtValue()); 2474 } 2475 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2476 } 2477 return {NewReg, Result}; 2478 } 2479 2480 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2481 const SPIRVType *ResType, 2482 MachineInstr &I) const { 2483 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2484 return selectCmp(ResVReg, ResType, CmpOp, I); 2485 } 2486 2487 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2488 MachineInstr &I) const { 2489 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2490 bool ZeroAsNull = STI.isOpenCLEnv(); 2491 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2492 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2493 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2494 } 2495 2496 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2497 MachineInstr &I) const { 2498 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2499 bool ZeroAsNull = STI.isOpenCLEnv(); 2500 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2501 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2502 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2503 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2504 } 2505 2506 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2507 MachineInstr &I) const { 2508 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2509 bool ZeroAsNull = STI.isOpenCLEnv(); 2510 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2511 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2512 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2513 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2514 } 2515 2516 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2517 const SPIRVType *ResType, 2518 MachineInstr &I) const { 2519 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2520 APInt One = 2521 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2522 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2523 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2524 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2525 } 2526 2527 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2528 const SPIRVType *ResType, 2529 MachineInstr &I, 2530 bool IsSigned) const { 2531 // To extend a bool, we need to use OpSelect between constants. 2532 Register ZeroReg = buildZerosVal(ResType, I); 2533 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2534 bool IsScalarBool = 2535 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2536 unsigned Opcode = 2537 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2538 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2539 .addDef(ResVReg) 2540 .addUse(GR.getSPIRVTypeID(ResType)) 2541 .addUse(I.getOperand(1).getReg()) 2542 .addUse(OneReg) 2543 .addUse(ZeroReg) 2544 .constrainAllUses(TII, TRI, RBI); 2545 } 2546 2547 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2548 const SPIRVType *ResType, 2549 MachineInstr &I, bool IsSigned, 2550 unsigned Opcode) const { 2551 Register SrcReg = I.getOperand(1).getReg(); 2552 // We can convert bool value directly to float type without OpConvert*ToF, 2553 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2554 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2555 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2556 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2557 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2558 const unsigned NumElts = ResType->getOperand(2).getImm(); 2559 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2560 } 2561 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2562 selectSelect(SrcReg, TmpType, I, false); 2563 } 2564 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2565 } 2566 2567 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2568 const SPIRVType *ResType, 2569 MachineInstr &I, bool IsSigned) const { 2570 Register SrcReg = I.getOperand(1).getReg(); 2571 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2572 return selectSelect(ResVReg, ResType, I, IsSigned); 2573 2574 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2575 if (SrcType == ResType) 2576 return BuildCOPY(ResVReg, SrcReg, I); 2577 2578 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2579 return selectUnOp(ResVReg, ResType, I, Opcode); 2580 } 2581 2582 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2583 const SPIRVType *ResType, 2584 MachineInstr &I, 2585 bool IsSigned) const { 2586 MachineIRBuilder MIRBuilder(I); 2587 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2588 MachineBasicBlock &BB = *I.getParent(); 2589 // Ensure we have bool. 2590 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2591 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2592 if (N > 1) 2593 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2594 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2595 // Build less-than-equal and less-than. 2596 // TODO: replace with one-liner createVirtualRegister() from 2597 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2598 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2599 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2600 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2601 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2602 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2603 : SPIRV::OpULessThanEqual)) 2604 .addDef(IsLessEqReg) 2605 .addUse(BoolTypeReg) 2606 .addUse(I.getOperand(1).getReg()) 2607 .addUse(I.getOperand(2).getReg()) 2608 .constrainAllUses(TII, TRI, RBI); 2609 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2610 MRI->setType(IsLessReg, LLT::scalar(64)); 2611 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2612 Result &= BuildMI(BB, I, I.getDebugLoc(), 2613 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2614 .addDef(IsLessReg) 2615 .addUse(BoolTypeReg) 2616 .addUse(I.getOperand(1).getReg()) 2617 .addUse(I.getOperand(2).getReg()) 2618 .constrainAllUses(TII, TRI, RBI); 2619 // Build selects. 2620 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2621 Register NegOneOrZeroReg = 2622 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2623 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2624 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2625 unsigned SelectOpcode = 2626 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2627 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2628 .addDef(NegOneOrZeroReg) 2629 .addUse(ResTypeReg) 2630 .addUse(IsLessReg) 2631 .addUse(buildOnesVal(true, ResType, I)) // -1 2632 .addUse(buildZerosVal(ResType, I)) 2633 .constrainAllUses(TII, TRI, RBI); 2634 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2635 .addDef(ResVReg) 2636 .addUse(ResTypeReg) 2637 .addUse(IsLessEqReg) 2638 .addUse(NegOneOrZeroReg) // -1 or 0 2639 .addUse(buildOnesVal(false, ResType, I)) 2640 .constrainAllUses(TII, TRI, RBI); 2641 } 2642 2643 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2644 Register ResVReg, 2645 MachineInstr &I, 2646 const SPIRVType *IntTy, 2647 const SPIRVType *BoolTy) const { 2648 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2649 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2650 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2651 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2652 Register Zero = buildZerosVal(IntTy, I); 2653 Register One = buildOnesVal(false, IntTy, I); 2654 MachineBasicBlock &BB = *I.getParent(); 2655 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2656 .addDef(BitIntReg) 2657 .addUse(GR.getSPIRVTypeID(IntTy)) 2658 .addUse(IntReg) 2659 .addUse(One) 2660 .constrainAllUses(TII, TRI, RBI); 2661 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2662 .addDef(ResVReg) 2663 .addUse(GR.getSPIRVTypeID(BoolTy)) 2664 .addUse(BitIntReg) 2665 .addUse(Zero) 2666 .constrainAllUses(TII, TRI, RBI); 2667 } 2668 2669 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2670 const SPIRVType *ResType, 2671 MachineInstr &I) const { 2672 Register IntReg = I.getOperand(1).getReg(); 2673 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2674 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2675 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2676 if (ArgType == ResType) 2677 return BuildCOPY(ResVReg, IntReg, I); 2678 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2679 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2680 return selectUnOp(ResVReg, ResType, I, Opcode); 2681 } 2682 2683 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2684 const SPIRVType *ResType, 2685 const APInt &Imm, 2686 MachineInstr &I) const { 2687 unsigned TyOpcode = ResType->getOpcode(); 2688 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2689 MachineBasicBlock &BB = *I.getParent(); 2690 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2691 Imm.isZero()) 2692 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2693 .addDef(ResVReg) 2694 .addUse(GR.getSPIRVTypeID(ResType)) 2695 .constrainAllUses(TII, TRI, RBI); 2696 if (TyOpcode == SPIRV::OpTypeInt) { 2697 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2698 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2699 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2700 } 2701 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2702 .addDef(ResVReg) 2703 .addUse(GR.getSPIRVTypeID(ResType)); 2704 // <=32-bit integers should be caught by the sdag pattern. 2705 assert(Imm.getBitWidth() > 32); 2706 addNumImm(Imm, MIB); 2707 return MIB.constrainAllUses(TII, TRI, RBI); 2708 } 2709 2710 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2711 const SPIRVType *ResType, 2712 MachineInstr &I) const { 2713 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2714 .addDef(ResVReg) 2715 .addUse(GR.getSPIRVTypeID(ResType)) 2716 .constrainAllUses(TII, TRI, RBI); 2717 } 2718 2719 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2720 assert(MO.isReg()); 2721 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2722 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2723 assert(TypeInst->getOperand(1).isReg()); 2724 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2725 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2726 } 2727 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2728 } 2729 2730 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2731 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2732 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2733 return TypeInst->getOperand(2).getImm(); 2734 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2735 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2736 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2737 } 2738 2739 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2740 const SPIRVType *ResType, 2741 MachineInstr &I) const { 2742 MachineBasicBlock &BB = *I.getParent(); 2743 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2744 .addDef(ResVReg) 2745 .addUse(GR.getSPIRVTypeID(ResType)) 2746 // object to insert 2747 .addUse(I.getOperand(3).getReg()) 2748 // composite to insert into 2749 .addUse(I.getOperand(2).getReg()); 2750 for (unsigned i = 4; i < I.getNumOperands(); i++) 2751 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2752 return MIB.constrainAllUses(TII, TRI, RBI); 2753 } 2754 2755 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2756 const SPIRVType *ResType, 2757 MachineInstr &I) const { 2758 MachineBasicBlock &BB = *I.getParent(); 2759 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2760 .addDef(ResVReg) 2761 .addUse(GR.getSPIRVTypeID(ResType)) 2762 .addUse(I.getOperand(2).getReg()); 2763 for (unsigned i = 3; i < I.getNumOperands(); i++) 2764 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2765 return MIB.constrainAllUses(TII, TRI, RBI); 2766 } 2767 2768 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2769 const SPIRVType *ResType, 2770 MachineInstr &I) const { 2771 if (isImm(I.getOperand(4), MRI)) 2772 return selectInsertVal(ResVReg, ResType, I); 2773 MachineBasicBlock &BB = *I.getParent(); 2774 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2775 .addDef(ResVReg) 2776 .addUse(GR.getSPIRVTypeID(ResType)) 2777 .addUse(I.getOperand(2).getReg()) 2778 .addUse(I.getOperand(3).getReg()) 2779 .addUse(I.getOperand(4).getReg()) 2780 .constrainAllUses(TII, TRI, RBI); 2781 } 2782 2783 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2784 const SPIRVType *ResType, 2785 MachineInstr &I) const { 2786 if (isImm(I.getOperand(3), MRI)) 2787 return selectExtractVal(ResVReg, ResType, I); 2788 MachineBasicBlock &BB = *I.getParent(); 2789 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2790 .addDef(ResVReg) 2791 .addUse(GR.getSPIRVTypeID(ResType)) 2792 .addUse(I.getOperand(2).getReg()) 2793 .addUse(I.getOperand(3).getReg()) 2794 .constrainAllUses(TII, TRI, RBI); 2795 } 2796 2797 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2798 const SPIRVType *ResType, 2799 MachineInstr &I) const { 2800 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2801 2802 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2803 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2804 // we have to use Op[InBounds]AccessChain. 2805 const unsigned Opcode = STI.isVulkanEnv() 2806 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2807 : SPIRV::OpAccessChain) 2808 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2809 : SPIRV::OpPtrAccessChain); 2810 2811 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2812 .addDef(ResVReg) 2813 .addUse(GR.getSPIRVTypeID(ResType)) 2814 // Object to get a pointer to. 2815 .addUse(I.getOperand(3).getReg()); 2816 // Adding indices. 2817 const unsigned StartingIndex = 2818 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2819 ? 5 2820 : 4; 2821 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2822 Res.addUse(I.getOperand(i).getReg()); 2823 return Res.constrainAllUses(TII, TRI, RBI); 2824 } 2825 2826 // Maybe wrap a value into OpSpecConstantOp 2827 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2828 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2829 bool Result = true; 2830 unsigned Lim = I.getNumExplicitOperands(); 2831 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2832 Register OpReg = I.getOperand(i).getReg(); 2833 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2834 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2835 SmallPtrSet<SPIRVType *, 4> Visited; 2836 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2837 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2838 GR.isAggregateType(OpType)) { 2839 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2840 // by selectAddrSpaceCast() 2841 CompositeArgs.push_back(OpReg); 2842 continue; 2843 } 2844 MachineFunction *MF = I.getMF(); 2845 Register WrapReg = GR.find(OpDefine, MF); 2846 if (WrapReg.isValid()) { 2847 CompositeArgs.push_back(WrapReg); 2848 continue; 2849 } 2850 // Create a new register for the wrapper 2851 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2852 GR.add(OpDefine, MF, WrapReg); 2853 CompositeArgs.push_back(WrapReg); 2854 // Decorate the wrapper register and generate a new instruction 2855 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2856 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2857 MachineBasicBlock &BB = *I.getParent(); 2858 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2859 .addDef(WrapReg) 2860 .addUse(GR.getSPIRVTypeID(OpType)) 2861 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2862 .addUse(OpReg) 2863 .constrainAllUses(TII, TRI, RBI); 2864 if (!Result) 2865 break; 2866 } 2867 return Result; 2868 } 2869 2870 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2871 const SPIRVType *ResType, 2872 MachineInstr &I) const { 2873 MachineBasicBlock &BB = *I.getParent(); 2874 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2875 switch (IID) { 2876 case Intrinsic::spv_load: 2877 return selectLoad(ResVReg, ResType, I); 2878 case Intrinsic::spv_store: 2879 return selectStore(I); 2880 case Intrinsic::spv_extractv: 2881 return selectExtractVal(ResVReg, ResType, I); 2882 case Intrinsic::spv_insertv: 2883 return selectInsertVal(ResVReg, ResType, I); 2884 case Intrinsic::spv_extractelt: 2885 return selectExtractElt(ResVReg, ResType, I); 2886 case Intrinsic::spv_insertelt: 2887 return selectInsertElt(ResVReg, ResType, I); 2888 case Intrinsic::spv_gep: 2889 return selectGEP(ResVReg, ResType, I); 2890 case Intrinsic::spv_unref_global: 2891 case Intrinsic::spv_init_global: { 2892 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2893 MachineInstr *Init = I.getNumExplicitOperands() > 2 2894 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2895 : nullptr; 2896 assert(MI); 2897 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2898 } 2899 case Intrinsic::spv_undef: { 2900 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2901 .addDef(ResVReg) 2902 .addUse(GR.getSPIRVTypeID(ResType)); 2903 return MIB.constrainAllUses(TII, TRI, RBI); 2904 } 2905 case Intrinsic::spv_const_composite: { 2906 // If no values are attached, the composite is null constant. 2907 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2908 // Select a proper instruction. 2909 unsigned Opcode = SPIRV::OpConstantNull; 2910 SmallVector<Register> CompositeArgs; 2911 if (!IsNull) { 2912 Opcode = SPIRV::OpConstantComposite; 2913 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2914 return false; 2915 } 2916 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2917 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2918 .addDef(ResVReg) 2919 .addUse(GR.getSPIRVTypeID(ResType)); 2920 // skip type MD node we already used when generated assign.type for this 2921 if (!IsNull) { 2922 for (Register OpReg : CompositeArgs) 2923 MIB.addUse(OpReg); 2924 } 2925 return MIB.constrainAllUses(TII, TRI, RBI); 2926 } 2927 case Intrinsic::spv_assign_name: { 2928 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2929 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2930 for (unsigned i = I.getNumExplicitDefs() + 2; 2931 i < I.getNumExplicitOperands(); ++i) { 2932 MIB.addImm(I.getOperand(i).getImm()); 2933 } 2934 return MIB.constrainAllUses(TII, TRI, RBI); 2935 } 2936 case Intrinsic::spv_switch: { 2937 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2938 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2939 if (I.getOperand(i).isReg()) 2940 MIB.addReg(I.getOperand(i).getReg()); 2941 else if (I.getOperand(i).isCImm()) 2942 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2943 else if (I.getOperand(i).isMBB()) 2944 MIB.addMBB(I.getOperand(i).getMBB()); 2945 else 2946 llvm_unreachable("Unexpected OpSwitch operand"); 2947 } 2948 return MIB.constrainAllUses(TII, TRI, RBI); 2949 } 2950 case Intrinsic::spv_loop_merge: { 2951 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge)); 2952 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2953 assert(I.getOperand(i).isMBB()); 2954 MIB.addMBB(I.getOperand(i).getMBB()); 2955 } 2956 MIB.addImm(SPIRV::SelectionControl::None); 2957 return MIB.constrainAllUses(TII, TRI, RBI); 2958 } 2959 case Intrinsic::spv_selection_merge: { 2960 auto MIB = 2961 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge)); 2962 assert(I.getOperand(1).isMBB() && 2963 "operand 1 to spv_selection_merge must be a basic block"); 2964 MIB.addMBB(I.getOperand(1).getMBB()); 2965 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm())); 2966 return MIB.constrainAllUses(TII, TRI, RBI); 2967 } 2968 case Intrinsic::spv_cmpxchg: 2969 return selectAtomicCmpXchg(ResVReg, ResType, I); 2970 case Intrinsic::spv_unreachable: 2971 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2972 .constrainAllUses(TII, TRI, RBI); 2973 case Intrinsic::spv_alloca: 2974 return selectFrameIndex(ResVReg, ResType, I); 2975 case Intrinsic::spv_alloca_array: 2976 return selectAllocaArray(ResVReg, ResType, I); 2977 case Intrinsic::spv_assume: 2978 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2979 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2980 .addUse(I.getOperand(1).getReg()) 2981 .constrainAllUses(TII, TRI, RBI); 2982 break; 2983 case Intrinsic::spv_expect: 2984 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2985 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2986 .addDef(ResVReg) 2987 .addUse(GR.getSPIRVTypeID(ResType)) 2988 .addUse(I.getOperand(2).getReg()) 2989 .addUse(I.getOperand(3).getReg()) 2990 .constrainAllUses(TII, TRI, RBI); 2991 break; 2992 case Intrinsic::arithmetic_fence: 2993 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2994 return BuildMI(BB, I, I.getDebugLoc(), 2995 TII.get(SPIRV::OpArithmeticFenceEXT)) 2996 .addDef(ResVReg) 2997 .addUse(GR.getSPIRVTypeID(ResType)) 2998 .addUse(I.getOperand(2).getReg()) 2999 .constrainAllUses(TII, TRI, RBI); 3000 else 3001 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 3002 break; 3003 case Intrinsic::spv_thread_id: 3004 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 3005 // intrinsic in LLVM IR for SPIR-V backend. 3006 // 3007 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 3008 // `GlobalInvocationId` builtin variable 3009 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 3010 ResType, I); 3011 case Intrinsic::spv_thread_id_in_group: 3012 // The HLSL SV_GroupThreadId semantic is lowered to 3013 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 3014 // 3015 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 3016 // translated to a `LocalInvocationId` builtin variable 3017 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 3018 ResType, I); 3019 case Intrinsic::spv_group_id: 3020 // The HLSL SV_GroupId semantic is lowered to 3021 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. 3022 // 3023 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` 3024 // builtin variable 3025 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, 3026 I); 3027 case Intrinsic::spv_fdot: 3028 return selectFloatDot(ResVReg, ResType, I); 3029 case Intrinsic::spv_udot: 3030 case Intrinsic::spv_sdot: 3031 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3032 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3033 return selectIntegerDot(ResVReg, ResType, I, 3034 /*Signed=*/IID == Intrinsic::spv_sdot); 3035 return selectIntegerDotExpansion(ResVReg, ResType, I); 3036 case Intrinsic::spv_dot4add_i8packed: 3037 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3038 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3039 return selectDot4AddPacked<true>(ResVReg, ResType, I); 3040 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 3041 case Intrinsic::spv_dot4add_u8packed: 3042 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3043 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3044 return selectDot4AddPacked<false>(ResVReg, ResType, I); 3045 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 3046 case Intrinsic::spv_all: 3047 return selectAll(ResVReg, ResType, I); 3048 case Intrinsic::spv_any: 3049 return selectAny(ResVReg, ResType, I); 3050 case Intrinsic::spv_cross: 3051 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 3052 case Intrinsic::spv_distance: 3053 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); 3054 case Intrinsic::spv_lerp: 3055 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 3056 case Intrinsic::spv_length: 3057 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 3058 case Intrinsic::spv_degrees: 3059 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 3060 case Intrinsic::spv_frac: 3061 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 3062 case Intrinsic::spv_normalize: 3063 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 3064 case Intrinsic::spv_rsqrt: 3065 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 3066 case Intrinsic::spv_sign: 3067 return selectSign(ResVReg, ResType, I); 3068 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 3069 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 3070 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 3071 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 3072 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb 3073 return selectFirstBitLow(ResVReg, ResType, I); 3074 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 3075 bool Result = true; 3076 auto MemSemConstant = 3077 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 3078 Register MemSemReg = MemSemConstant.first; 3079 Result &= MemSemConstant.second; 3080 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 3081 Register ScopeReg = ScopeConstant.first; 3082 Result &= ScopeConstant.second; 3083 MachineBasicBlock &BB = *I.getParent(); 3084 return Result && 3085 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 3086 .addUse(ScopeReg) 3087 .addUse(ScopeReg) 3088 .addUse(MemSemReg) 3089 .constrainAllUses(TII, TRI, RBI); 3090 } 3091 case Intrinsic::spv_lifetime_start: 3092 case Intrinsic::spv_lifetime_end: { 3093 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 3094 : SPIRV::OpLifetimeStop; 3095 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 3096 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 3097 if (Size == -1) 3098 Size = 0; 3099 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 3100 .addUse(PtrReg) 3101 .addImm(Size) 3102 .constrainAllUses(TII, TRI, RBI); 3103 } 3104 case Intrinsic::spv_saturate: 3105 return selectSaturate(ResVReg, ResType, I); 3106 case Intrinsic::spv_nclamp: 3107 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 3108 case Intrinsic::spv_uclamp: 3109 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 3110 case Intrinsic::spv_sclamp: 3111 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 3112 case Intrinsic::spv_wave_active_countbits: 3113 return selectWaveActiveCountBits(ResVReg, ResType, I); 3114 case Intrinsic::spv_wave_all: 3115 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 3116 case Intrinsic::spv_wave_any: 3117 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 3118 case Intrinsic::spv_wave_is_first_lane: 3119 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 3120 case Intrinsic::spv_wave_reduce_umax: 3121 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true); 3122 case Intrinsic::spv_wave_reduce_max: 3123 return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false); 3124 case Intrinsic::spv_wave_reduce_sum: 3125 return selectWaveReduceSum(ResVReg, ResType, I); 3126 case Intrinsic::spv_wave_readlane: 3127 return selectWaveOpInst(ResVReg, ResType, I, 3128 SPIRV::OpGroupNonUniformShuffle); 3129 case Intrinsic::spv_step: 3130 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 3131 case Intrinsic::spv_radians: 3132 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 3133 // Discard intrinsics which we do not expect to actually represent code after 3134 // lowering or intrinsics which are not implemented but should not crash when 3135 // found in a customer's LLVM IR input. 3136 case Intrinsic::instrprof_increment: 3137 case Intrinsic::instrprof_increment_step: 3138 case Intrinsic::instrprof_value_profile: 3139 break; 3140 // Discard internal intrinsics. 3141 case Intrinsic::spv_value_md: 3142 break; 3143 case Intrinsic::spv_resource_handlefrombinding: { 3144 return selectHandleFromBinding(ResVReg, ResType, I); 3145 } 3146 case Intrinsic::spv_resource_store_typedbuffer: { 3147 return selectImageWriteIntrinsic(I); 3148 } 3149 case Intrinsic::spv_resource_load_typedbuffer: { 3150 return selectReadImageIntrinsic(ResVReg, ResType, I); 3151 } 3152 case Intrinsic::spv_resource_getpointer: { 3153 return selectResourceGetPointer(ResVReg, ResType, I); 3154 } 3155 case Intrinsic::spv_discard: { 3156 return selectDiscard(ResVReg, ResType, I); 3157 } 3158 default: { 3159 std::string DiagMsg; 3160 raw_string_ostream OS(DiagMsg); 3161 I.print(OS); 3162 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3163 report_fatal_error(DiagMsg.c_str(), false); 3164 } 3165 } 3166 return true; 3167 } 3168 3169 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3170 const SPIRVType *ResType, 3171 MachineInstr &I) const { 3172 return true; 3173 } 3174 3175 bool SPIRVInstructionSelector::selectReadImageIntrinsic( 3176 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3177 3178 // If the load of the image is in a different basic block, then 3179 // this will generate invalid code. A proper solution is to move 3180 // the OpLoad from selectHandleFromBinding here. However, to do 3181 // that we will need to change the return type of the intrinsic. 3182 // We will do that when we can, but for now trying to move forward with other 3183 // issues. 3184 Register ImageReg = I.getOperand(2).getReg(); 3185 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 3186 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 3187 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 3188 *ImageDef, I)) { 3189 return false; 3190 } 3191 3192 Register IdxReg = I.getOperand(3).getReg(); 3193 DebugLoc Loc = I.getDebugLoc(); 3194 MachineInstr &Pos = I; 3195 3196 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, Loc, Pos); 3197 } 3198 3199 bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg, 3200 const SPIRVType *ResType, 3201 Register ImageReg, 3202 Register IdxReg, DebugLoc Loc, 3203 MachineInstr &Pos) const { 3204 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3205 if (ResultSize == 4) { 3206 return BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead)) 3207 .addDef(ResVReg) 3208 .addUse(GR.getSPIRVTypeID(ResType)) 3209 .addUse(ImageReg) 3210 .addUse(IdxReg) 3211 .constrainAllUses(TII, TRI, RBI); 3212 } 3213 3214 SPIRVType *ReadType = widenTypeToVec4(ResType, Pos); 3215 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3216 bool Succeed = 3217 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead)) 3218 .addDef(ReadReg) 3219 .addUse(GR.getSPIRVTypeID(ReadType)) 3220 .addUse(ImageReg) 3221 .addUse(IdxReg) 3222 .constrainAllUses(TII, TRI, RBI); 3223 if (!Succeed) 3224 return false; 3225 3226 if (ResultSize == 1) { 3227 return BuildMI(*Pos.getParent(), Pos, Loc, 3228 TII.get(SPIRV::OpCompositeExtract)) 3229 .addDef(ResVReg) 3230 .addUse(GR.getSPIRVTypeID(ResType)) 3231 .addUse(ReadReg) 3232 .addImm(0) 3233 .constrainAllUses(TII, TRI, RBI); 3234 } 3235 return extractSubvector(ResVReg, ResType, ReadReg, Pos); 3236 } 3237 3238 bool SPIRVInstructionSelector::selectResourceGetPointer( 3239 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3240 #ifdef ASSERT 3241 // For now, the operand is an image. This will change once we start handling 3242 // more resource types. 3243 Register ResourcePtr = I.getOperand(2).getReg(); 3244 SPIRVType *RegType = GR.getResultType(ResourcePtr); 3245 assert(RegType->getOpcode() == SPIRV::OpTypeImage && 3246 "Can only handle texel buffers for now."); 3247 #endif 3248 3249 // For texel buffers, the index into the image is part of the OpImageRead or 3250 // OpImageWrite instructions. So we will do nothing in this case. This 3251 // intrinsic will be combined with the load or store when selecting the load 3252 // or store. 3253 return true; 3254 } 3255 3256 bool SPIRVInstructionSelector::extractSubvector( 3257 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3258 MachineInstr &InsertionPoint) const { 3259 SPIRVType *InputType = GR.getResultType(ReadReg); 3260 [[maybe_unused]] uint64_t InputSize = 3261 GR.getScalarOrVectorComponentCount(InputType); 3262 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3263 assert(InputSize > 1 && "The input must be a vector."); 3264 assert(ResultSize > 1 && "The result must be a vector."); 3265 assert(ResultSize < InputSize && 3266 "Cannot extract more element than there are in the input."); 3267 SmallVector<Register> ComponentRegisters; 3268 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3269 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3270 for (uint64_t I = 0; I < ResultSize; I++) { 3271 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3272 bool Succeed = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3273 InsertionPoint.getDebugLoc(), 3274 TII.get(SPIRV::OpCompositeExtract)) 3275 .addDef(ComponentReg) 3276 .addUse(ScalarType->getOperand(0).getReg()) 3277 .addUse(ReadReg) 3278 .addImm(I) 3279 .constrainAllUses(TII, TRI, RBI); 3280 if (!Succeed) 3281 return false; 3282 ComponentRegisters.emplace_back(ComponentReg); 3283 } 3284 3285 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3286 InsertionPoint.getDebugLoc(), 3287 TII.get(SPIRV::OpCompositeConstruct)) 3288 .addDef(ResVReg) 3289 .addUse(GR.getSPIRVTypeID(ResType)); 3290 3291 for (Register ComponentReg : ComponentRegisters) 3292 MIB.addUse(ComponentReg); 3293 return MIB.constrainAllUses(TII, TRI, RBI); 3294 } 3295 3296 bool SPIRVInstructionSelector::selectImageWriteIntrinsic( 3297 MachineInstr &I) const { 3298 // If the load of the image is in a different basic block, then 3299 // this will generate invalid code. A proper solution is to move 3300 // the OpLoad from selectHandleFromBinding here. However, to do 3301 // that we will need to change the return type of the intrinsic. 3302 // We will do that when we can, but for now trying to move forward with other 3303 // issues. 3304 Register ImageReg = I.getOperand(1).getReg(); 3305 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 3306 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 3307 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 3308 *ImageDef, I)) { 3309 return false; 3310 } 3311 3312 Register CoordinateReg = I.getOperand(2).getReg(); 3313 Register DataReg = I.getOperand(3).getReg(); 3314 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3315 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3316 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3317 TII.get(SPIRV::OpImageWrite)) 3318 .addUse(NewImageReg) 3319 .addUse(CoordinateReg) 3320 .addUse(DataReg) 3321 .constrainAllUses(TII, TRI, RBI); 3322 } 3323 3324 Register SPIRVInstructionSelector::buildPointerToResource( 3325 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3326 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3327 MachineIRBuilder MIRBuilder) const { 3328 if (ArraySize == 1) 3329 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3330 MIRBuilder); 3331 3332 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3333 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3334 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3335 VarType, Set, Binding, MIRBuilder); 3336 3337 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3338 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3339 3340 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3341 if (IsNonUniform) { 3342 // It is unclear which value needs to be marked an non-uniform, so both 3343 // the index and the access changed are decorated as non-uniform. 3344 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3345 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3346 } 3347 3348 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3349 .addDef(AcReg) 3350 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3351 .addUse(VarReg) 3352 .addUse(IndexReg); 3353 3354 return AcReg; 3355 } 3356 3357 bool SPIRVInstructionSelector::selectFirstBitSet16( 3358 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3359 unsigned ExtendOpcode, unsigned BitSetOpcode) const { 3360 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3361 bool Result = selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, 3362 ExtendOpcode); 3363 3364 return Result && 3365 selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode); 3366 } 3367 3368 bool SPIRVInstructionSelector::selectFirstBitSet32( 3369 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3370 Register SrcReg, unsigned BitSetOpcode) const { 3371 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3372 .addDef(ResVReg) 3373 .addUse(GR.getSPIRVTypeID(ResType)) 3374 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3375 .addImm(BitSetOpcode) 3376 .addUse(SrcReg) 3377 .constrainAllUses(TII, TRI, RBI); 3378 } 3379 3380 bool SPIRVInstructionSelector::selectFirstBitSet64Overflow( 3381 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3382 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3383 3384 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors 3385 // requires creating a param register and return register with an invalid 3386 // vector size. If that is resolved, then this function can be used for 3387 // vectors of any component size. 3388 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3389 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops"); 3390 3391 MachineIRBuilder MIRBuilder(I); 3392 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3393 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder); 3394 SPIRVType *I64x2Type = GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder); 3395 SPIRVType *Vec2ResType = 3396 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder); 3397 3398 std::vector<Register> PartialRegs; 3399 3400 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd 3401 unsigned CurrentComponent = 0; 3402 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) { 3403 // This register holds the firstbitX result for each of the i64x2 vectors 3404 // extracted from SrcReg 3405 Register BitSetResult = 3406 MRI->createVirtualRegister(GR.getRegClass(I64x2Type)); 3407 3408 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3409 TII.get(SPIRV::OpVectorShuffle)) 3410 .addDef(BitSetResult) 3411 .addUse(GR.getSPIRVTypeID(I64x2Type)) 3412 .addUse(SrcReg) 3413 .addUse(SrcReg) 3414 .addImm(CurrentComponent) 3415 .addImm(CurrentComponent + 1); 3416 3417 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3418 return false; 3419 3420 Register SubVecBitSetReg = 3421 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType)); 3422 3423 if (!selectFirstBitSet64(SubVecBitSetReg, Vec2ResType, I, BitSetResult, 3424 BitSetOpcode, SwapPrimarySide)) 3425 return false; 3426 3427 PartialRegs.push_back(SubVecBitSetReg); 3428 } 3429 3430 // On odd component counts we need to handle one more component 3431 if (CurrentComponent != ComponentCount) { 3432 bool ZeroAsNull = STI.isOpenCLEnv(); 3433 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type)); 3434 Register ConstIntLastIdx = GR.getOrCreateConstInt( 3435 ComponentCount - 1, I, BaseType, TII, ZeroAsNull); 3436 3437 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx}, 3438 SPIRV::OpVectorExtractDynamic)) 3439 return false; 3440 3441 Register FinalElemBitSetReg = 3442 MRI->createVirtualRegister(GR.getRegClass(BaseType)); 3443 3444 if (!selectFirstBitSet64(FinalElemBitSetReg, BaseType, I, FinalElemReg, 3445 BitSetOpcode, SwapPrimarySide)) 3446 return false; 3447 3448 PartialRegs.push_back(FinalElemBitSetReg); 3449 } 3450 3451 // Join all the resulting registers back into the return type in order 3452 // (ie i32x2, i32x2, i32x1 -> i32x5) 3453 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs, 3454 SPIRV::OpCompositeConstruct); 3455 } 3456 3457 bool SPIRVInstructionSelector::selectFirstBitSet64( 3458 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3459 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3460 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3461 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3462 bool ZeroAsNull = STI.isOpenCLEnv(); 3463 Register ConstIntZero = 3464 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull); 3465 Register ConstIntOne = 3466 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull); 3467 3468 // SPIRV doesn't support vectors with more than 4 components. Since the 3469 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only 3470 // operate on vectors with 2 or less components. When largers vectors are 3471 // seen. Split them, recurse, then recombine them. 3472 if (ComponentCount > 2) { 3473 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg, 3474 BitSetOpcode, SwapPrimarySide); 3475 } 3476 3477 // 1. Split int64 into 2 pieces using a bitcast 3478 MachineIRBuilder MIRBuilder(I); 3479 SPIRVType *PostCastType = 3480 GR.getOrCreateSPIRVVectorType(BaseType, 2 * ComponentCount, MIRBuilder); 3481 Register BitcastReg = 3482 MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3483 3484 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg}, 3485 SPIRV::OpBitcast)) 3486 return false; 3487 3488 // 2. Find the first set bit from the primary side for all the pieces in #1 3489 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3490 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode)) 3491 return false; 3492 3493 // 3. Split result vector into high bits and low bits 3494 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3495 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3496 3497 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3498 if (IsScalarRes) { 3499 // if scalar do a vector extract 3500 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntZero}, 3501 SPIRV::OpVectorExtractDynamic)) 3502 return false; 3503 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntOne}, 3504 SPIRV::OpVectorExtractDynamic)) 3505 return false; 3506 } else { 3507 // if vector do a shufflevector 3508 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3509 TII.get(SPIRV::OpVectorShuffle)) 3510 .addDef(HighReg) 3511 .addUse(GR.getSPIRVTypeID(ResType)) 3512 .addUse(FBSReg) 3513 // Per the spec, repeat the vector if only one vec is needed 3514 .addUse(FBSReg); 3515 3516 // high bits are stored in even indexes. Extract them from FBSReg 3517 for (unsigned J = 0; J < ComponentCount * 2; J += 2) { 3518 MIB.addImm(J); 3519 } 3520 3521 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3522 return false; 3523 3524 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3525 TII.get(SPIRV::OpVectorShuffle)) 3526 .addDef(LowReg) 3527 .addUse(GR.getSPIRVTypeID(ResType)) 3528 .addUse(FBSReg) 3529 // Per the spec, repeat the vector if only one vec is needed 3530 .addUse(FBSReg); 3531 3532 // low bits are stored in odd indexes. Extract them from FBSReg 3533 for (unsigned J = 1; J < ComponentCount * 2; J += 2) { 3534 MIB.addImm(J); 3535 } 3536 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3537 return false; 3538 } 3539 3540 // 4. Check the result. When primary bits == -1 use secondary, otherwise use 3541 // primary 3542 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3543 Register NegOneReg; 3544 Register Reg0; 3545 Register Reg32; 3546 unsigned SelectOp; 3547 unsigned AddOp; 3548 3549 if (IsScalarRes) { 3550 NegOneReg = 3551 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3552 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3553 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3554 SelectOp = SPIRV::OpSelectSISCond; 3555 AddOp = SPIRV::OpIAddS; 3556 } else { 3557 BoolType = 3558 GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount, MIRBuilder); 3559 NegOneReg = 3560 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3561 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3562 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3563 SelectOp = SPIRV::OpSelectVIVCond; 3564 AddOp = SPIRV::OpIAddV; 3565 } 3566 3567 Register PrimaryReg = HighReg; 3568 Register SecondaryReg = LowReg; 3569 Register PrimaryShiftReg = Reg32; 3570 Register SecondaryShiftReg = Reg0; 3571 3572 // By default the emitted opcodes check for the set bit from the MSB side. 3573 // Setting SwapPrimarySide checks the set bit from the LSB side 3574 if (SwapPrimarySide) { 3575 PrimaryReg = LowReg; 3576 SecondaryReg = HighReg; 3577 PrimaryShiftReg = Reg0; 3578 SecondaryShiftReg = Reg32; 3579 } 3580 3581 // Check if the primary bits are == -1 3582 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3583 if (!selectOpWithSrcs(BReg, BoolType, I, {PrimaryReg, NegOneReg}, 3584 SPIRV::OpIEqual)) 3585 return false; 3586 3587 // Select secondary bits if true in BReg, otherwise primary bits 3588 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3589 if (!selectOpWithSrcs(TmpReg, ResType, I, {BReg, SecondaryReg, PrimaryReg}, 3590 SelectOp)) 3591 return false; 3592 3593 // 5. Add 32 when high bits are used, otherwise 0 for low bits 3594 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3595 if (!selectOpWithSrcs(ValReg, ResType, I, 3596 {BReg, SecondaryShiftReg, PrimaryShiftReg}, SelectOp)) 3597 return false; 3598 3599 return selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, AddOp); 3600 } 3601 3602 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3603 const SPIRVType *ResType, 3604 MachineInstr &I, 3605 bool IsSigned) const { 3606 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3607 Register OpReg = I.getOperand(2).getReg(); 3608 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3609 // zero or sign extend 3610 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3611 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3612 3613 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3614 case 16: 3615 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3616 case 32: 3617 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3618 case 64: 3619 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3620 /*SwapPrimarySide=*/false); 3621 default: 3622 report_fatal_error( 3623 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3624 } 3625 } 3626 3627 bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg, 3628 const SPIRVType *ResType, 3629 MachineInstr &I) const { 3630 // FindILsb intrinsic only supports 32 bit integers 3631 Register OpReg = I.getOperand(2).getReg(); 3632 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3633 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it 3634 // to an unsigned i32. As this leaves all the least significant bits unchanged 3635 // so the first set bit from the LSB side doesn't change. 3636 unsigned ExtendOpcode = SPIRV::OpUConvert; 3637 unsigned BitSetOpcode = GL::FindILsb; 3638 3639 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3640 case 16: 3641 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3642 case 32: 3643 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3644 case 64: 3645 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3646 /*SwapPrimarySide=*/true); 3647 default: 3648 report_fatal_error("spv_firstbitlow only supports 16,32,64 bits."); 3649 } 3650 } 3651 3652 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3653 const SPIRVType *ResType, 3654 MachineInstr &I) const { 3655 // there was an allocation size parameter to the allocation instruction 3656 // that is not 1 3657 MachineBasicBlock &BB = *I.getParent(); 3658 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3659 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3660 .addDef(ResVReg) 3661 .addUse(GR.getSPIRVTypeID(ResType)) 3662 .addUse(I.getOperand(2).getReg()) 3663 .constrainAllUses(TII, TRI, RBI); 3664 if (!STI.isVulkanEnv()) { 3665 unsigned Alignment = I.getOperand(3).getImm(); 3666 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3667 } 3668 return Res; 3669 } 3670 3671 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3672 const SPIRVType *ResType, 3673 MachineInstr &I) const { 3674 // Change order of instructions if needed: all OpVariable instructions in a 3675 // function must be the first instructions in the first block 3676 auto It = getOpVariableMBBIt(I); 3677 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3678 TII.get(SPIRV::OpVariable)) 3679 .addDef(ResVReg) 3680 .addUse(GR.getSPIRVTypeID(ResType)) 3681 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3682 .constrainAllUses(TII, TRI, RBI); 3683 if (!STI.isVulkanEnv()) { 3684 unsigned Alignment = I.getOperand(2).getImm(); 3685 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3686 {Alignment}); 3687 } 3688 return Res; 3689 } 3690 3691 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3692 // InstructionSelector walks backwards through the instructions. We can use 3693 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3694 // first, so can generate an OpBranchConditional here. If there is no 3695 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3696 const MachineInstr *PrevI = I.getPrevNode(); 3697 MachineBasicBlock &MBB = *I.getParent(); 3698 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3699 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3700 .addUse(PrevI->getOperand(0).getReg()) 3701 .addMBB(PrevI->getOperand(1).getMBB()) 3702 .addMBB(I.getOperand(0).getMBB()) 3703 .constrainAllUses(TII, TRI, RBI); 3704 } 3705 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3706 .addMBB(I.getOperand(0).getMBB()) 3707 .constrainAllUses(TII, TRI, RBI); 3708 } 3709 3710 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3711 // InstructionSelector walks backwards through the instructions. For an 3712 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3713 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3714 // generate the OpBranchConditional in selectBranch above. 3715 // 3716 // If an OpBranchConditional has been generated, we simply return, as the work 3717 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3718 // implicit fallthrough to the next basic block, so we need to create an 3719 // OpBranchConditional with an explicit "false" argument pointing to the next 3720 // basic block that LLVM would fall through to. 3721 const MachineInstr *NextI = I.getNextNode(); 3722 // Check if this has already been successfully selected. 3723 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3724 return true; 3725 // Must be relying on implicit block fallthrough, so generate an 3726 // OpBranchConditional with the "next" basic block as the "false" target. 3727 MachineBasicBlock &MBB = *I.getParent(); 3728 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3729 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3730 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3731 .addUse(I.getOperand(0).getReg()) 3732 .addMBB(I.getOperand(1).getMBB()) 3733 .addMBB(NextMBB) 3734 .constrainAllUses(TII, TRI, RBI); 3735 } 3736 3737 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3738 const SPIRVType *ResType, 3739 MachineInstr &I) const { 3740 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3741 .addDef(ResVReg) 3742 .addUse(GR.getSPIRVTypeID(ResType)); 3743 const unsigned NumOps = I.getNumOperands(); 3744 for (unsigned i = 1; i < NumOps; i += 2) { 3745 MIB.addUse(I.getOperand(i + 0).getReg()); 3746 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3747 } 3748 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3749 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3750 MIB->removeOperand(1); 3751 return Res; 3752 } 3753 3754 bool SPIRVInstructionSelector::selectGlobalValue( 3755 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3756 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3757 MachineIRBuilder MIRBuilder(I); 3758 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3759 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3760 SPIRVType *PointerBaseType; 3761 if (GVType->isArrayTy()) { 3762 SPIRVType *ArrayElementType = 3763 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3764 SPIRV::AccessQualifier::ReadWrite, false); 3765 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3766 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3767 } else { 3768 PointerBaseType = GR.getOrCreateSPIRVType( 3769 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3770 } 3771 3772 std::string GlobalIdent; 3773 if (!GV->hasName()) { 3774 unsigned &ID = UnnamedGlobalIDs[GV]; 3775 if (ID == 0) 3776 ID = UnnamedGlobalIDs.size(); 3777 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3778 } else { 3779 GlobalIdent = GV->getName(); 3780 } 3781 3782 // Behaviour of functions as operands depends on availability of the 3783 // corresponding extension (SPV_INTEL_function_pointers): 3784 // - If there is an extension to operate with functions as operands: 3785 // We create a proper constant operand and evaluate a correct type for a 3786 // function pointer. 3787 // - Without the required extension: 3788 // We have functions as operands in tests with blocks of instruction e.g. in 3789 // transcoding/global_block.ll. These operands are not used and should be 3790 // substituted by zero constants. Their type is expected to be always 3791 // OpTypePointer Function %uchar. 3792 if (isa<Function>(GV)) { 3793 const Constant *ConstVal = GV; 3794 MachineBasicBlock &BB = *I.getParent(); 3795 Register NewReg = GR.find(ConstVal, GR.CurMF); 3796 if (!NewReg.isValid()) { 3797 Register NewReg = ResVReg; 3798 GR.add(ConstVal, GR.CurMF, NewReg); 3799 const Function *GVFun = 3800 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3801 ? dyn_cast<Function>(GV) 3802 : nullptr; 3803 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3804 PointerBaseType, I, TII, 3805 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3806 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3807 if (GVFun) { 3808 // References to a function via function pointers generate virtual 3809 // registers without a definition. We will resolve it later, during 3810 // module analysis stage. 3811 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 3812 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3813 Register FuncVReg = 3814 MRI->createGenericVirtualRegister(GR.getRegType(ResType)); 3815 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); 3816 MachineInstrBuilder MIB1 = 3817 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 3818 .addDef(FuncVReg) 3819 .addUse(ResTypeReg); 3820 MachineInstrBuilder MIB2 = 3821 BuildMI(BB, I, I.getDebugLoc(), 3822 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3823 .addDef(NewReg) 3824 .addUse(ResTypeReg) 3825 .addUse(FuncVReg); 3826 // mapping the function pointer to the used Function 3827 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); 3828 return MIB1.constrainAllUses(TII, TRI, RBI) && 3829 MIB2.constrainAllUses(TII, TRI, RBI); 3830 } 3831 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3832 .addDef(NewReg) 3833 .addUse(GR.getSPIRVTypeID(ResType)) 3834 .constrainAllUses(TII, TRI, RBI); 3835 } 3836 assert(NewReg != ResVReg); 3837 return BuildCOPY(ResVReg, NewReg, I); 3838 } 3839 auto GlobalVar = cast<GlobalVariable>(GV); 3840 assert(GlobalVar->getName() != "llvm.global.annotations"); 3841 3842 // Skip empty declaration for GVs with initializers till we get the decl with 3843 // passed initializer. 3844 if (hasInitializer(GlobalVar) && !Init) 3845 return true; 3846 3847 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage(); 3848 SPIRV::LinkageType::LinkageType LnkType = 3849 GV->isDeclarationForLinker() 3850 ? SPIRV::LinkageType::Import 3851 : (GV->hasLinkOnceODRLinkage() && 3852 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3853 ? SPIRV::LinkageType::LinkOnceODR 3854 : SPIRV::LinkageType::Export); 3855 3856 const unsigned AddrSpace = GV->getAddressSpace(); 3857 SPIRV::StorageClass::StorageClass StorageClass = 3858 addressSpaceToStorageClass(AddrSpace, STI); 3859 SPIRVType *ResType = 3860 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3861 Register Reg = GR.buildGlobalVariable( 3862 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3863 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3864 return Reg.isValid(); 3865 } 3866 3867 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3868 const SPIRVType *ResType, 3869 MachineInstr &I) const { 3870 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3871 return selectExtInst(ResVReg, ResType, I, CL::log10); 3872 } 3873 3874 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3875 // is implemented as: 3876 // log10(x) = log2(x) * (1 / log2(10)) 3877 // = log2(x) * 0.30103 3878 3879 MachineIRBuilder MIRBuilder(I); 3880 MachineBasicBlock &BB = *I.getParent(); 3881 3882 // Build log2(x). 3883 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3884 bool Result = 3885 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3886 .addDef(VarReg) 3887 .addUse(GR.getSPIRVTypeID(ResType)) 3888 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3889 .addImm(GL::Log2) 3890 .add(I.getOperand(1)) 3891 .constrainAllUses(TII, TRI, RBI); 3892 3893 // Build 0.30103. 3894 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3895 ResType->getOpcode() == SPIRV::OpTypeFloat); 3896 // TODO: Add matrix implementation once supported by the HLSL frontend. 3897 const SPIRVType *SpirvScalarType = 3898 ResType->getOpcode() == SPIRV::OpTypeVector 3899 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3900 : ResType; 3901 Register ScaleReg = 3902 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3903 3904 // Multiply log2(x) by 0.30103 to get log10(x) result. 3905 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3906 ? SPIRV::OpVectorTimesScalar 3907 : SPIRV::OpFMulS; 3908 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3909 .addDef(ResVReg) 3910 .addUse(GR.getSPIRVTypeID(ResType)) 3911 .addUse(VarReg) 3912 .addUse(ScaleReg) 3913 .constrainAllUses(TII, TRI, RBI); 3914 } 3915 3916 // Generate the instructions to load 3-element vector builtin input 3917 // IDs/Indices. 3918 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3919 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3920 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3921 const SPIRVType *ResType, MachineInstr &I) const { 3922 MachineIRBuilder MIRBuilder(I); 3923 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3924 const SPIRVType *Vec3Ty = 3925 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3926 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3927 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3928 3929 // Create new register for the input ID builtin variable. 3930 Register NewRegister = 3931 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3932 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3933 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3934 3935 // Build global variable with the necessary decorations for the input ID 3936 // builtin variable. 3937 Register Variable = GR.buildGlobalVariable( 3938 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3939 SPIRV::StorageClass::Input, nullptr, true, true, 3940 SPIRV::LinkageType::Import, MIRBuilder, false); 3941 3942 // Create new register for loading value. 3943 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3944 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3945 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3946 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3947 3948 // Load v3uint value from the global variable. 3949 bool Result = 3950 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3951 .addDef(LoadedRegister) 3952 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3953 .addUse(Variable); 3954 3955 // Get the input ID index. Expecting operand is a constant immediate value, 3956 // wrapped in a type assignment. 3957 assert(I.getOperand(2).isReg()); 3958 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3959 3960 // Extract the input ID from the loaded vector value. 3961 MachineBasicBlock &BB = *I.getParent(); 3962 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3963 .addDef(ResVReg) 3964 .addUse(GR.getSPIRVTypeID(ResType)) 3965 .addUse(LoadedRegister) 3966 .addImm(ThreadId); 3967 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3968 } 3969 3970 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3971 MachineInstr &I) const { 3972 MachineIRBuilder MIRBuilder(I); 3973 if (Type->getOpcode() != SPIRV::OpTypeVector) 3974 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3975 3976 uint64_t VectorSize = Type->getOperand(2).getImm(); 3977 if (VectorSize == 4) 3978 return Type; 3979 3980 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3981 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3982 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3983 } 3984 3985 bool SPIRVInstructionSelector::loadHandleBeforePosition( 3986 Register &HandleReg, const SPIRVType *ResType, GIntrinsic &HandleDef, 3987 MachineInstr &Pos) const { 3988 3989 assert(HandleDef.getIntrinsicID() == 3990 Intrinsic::spv_resource_handlefrombinding); 3991 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI); 3992 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI); 3993 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI); 3994 Register IndexReg = HandleDef.getOperand(5).getReg(); 3995 bool IsNonUniform = ArraySize > 1 && foldImm(HandleDef.getOperand(6), MRI); 3996 3997 MachineIRBuilder MIRBuilder(HandleDef); 3998 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3999 IndexReg, IsNonUniform, MIRBuilder); 4000 4001 if (IsNonUniform) 4002 buildOpDecorate(HandleReg, HandleDef, TII, SPIRV::Decoration::NonUniformEXT, 4003 {}); 4004 4005 // TODO: For now we assume the resource is an image, which needs to be 4006 // loaded to get the handle. That will not be true for storage buffers. 4007 return BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), 4008 TII.get(SPIRV::OpLoad)) 4009 .addDef(HandleReg) 4010 .addUse(GR.getSPIRVTypeID(ResType)) 4011 .addUse(VarReg) 4012 .constrainAllUses(TII, TRI, RBI); 4013 } 4014 4015 namespace llvm { 4016 InstructionSelector * 4017 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 4018 const SPIRVSubtarget &Subtarget, 4019 const RegisterBankInfo &RBI) { 4020 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 4021 } 4022 } // namespace llvm 4023