1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 #include "llvm/Support/ErrorHandling.h" 37 38 #define DEBUG_TYPE "spirv-isel" 39 40 using namespace llvm; 41 namespace CL = SPIRV::OpenCLExtInst; 42 namespace GL = SPIRV::GLSLExtInst; 43 44 using ExtInstList = 45 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 46 47 namespace { 48 49 llvm::SPIRV::SelectionControl::SelectionControl 50 getSelectionOperandForImm(int Imm) { 51 if (Imm == 2) 52 return SPIRV::SelectionControl::Flatten; 53 if (Imm == 1) 54 return SPIRV::SelectionControl::DontFlatten; 55 if (Imm == 0) 56 return SPIRV::SelectionControl::None; 57 llvm_unreachable("Invalid immediate"); 58 } 59 60 #define GET_GLOBALISEL_PREDICATE_BITSET 61 #include "SPIRVGenGlobalISel.inc" 62 #undef GET_GLOBALISEL_PREDICATE_BITSET 63 64 class SPIRVInstructionSelector : public InstructionSelector { 65 const SPIRVSubtarget &STI; 66 const SPIRVInstrInfo &TII; 67 const SPIRVRegisterInfo &TRI; 68 const RegisterBankInfo &RBI; 69 SPIRVGlobalRegistry &GR; 70 MachineRegisterInfo *MRI; 71 MachineFunction *HasVRegsReset = nullptr; 72 73 /// We need to keep track of the number we give to anonymous global values to 74 /// generate the same name every time when this is needed. 75 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 76 SmallPtrSet<MachineInstr *, 8> DeadMIs; 77 78 public: 79 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 80 const SPIRVSubtarget &ST, 81 const RegisterBankInfo &RBI); 82 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 83 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 84 BlockFrequencyInfo *BFI) override; 85 // Common selection code. Instruction-specific selection occurs in spvSelect. 86 bool select(MachineInstr &I) override; 87 static const char *getName() { return DEBUG_TYPE; } 88 89 #define GET_GLOBALISEL_PREDICATES_DECL 90 #include "SPIRVGenGlobalISel.inc" 91 #undef GET_GLOBALISEL_PREDICATES_DECL 92 93 #define GET_GLOBALISEL_TEMPORARIES_DECL 94 #include "SPIRVGenGlobalISel.inc" 95 #undef GET_GLOBALISEL_TEMPORARIES_DECL 96 97 private: 98 void resetVRegsType(MachineFunction &MF); 99 100 // tblgen-erated 'select' implementation, used as the initial selector for 101 // the patterns that don't require complex C++. 102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 103 104 // All instruction-specific selection that didn't happen in "select()". 105 // Is basically a large Switch/Case delegating to all other select method. 106 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I) const; 108 109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 110 MachineInstr &I, bool IsSigned) const; 111 112 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I) const; 114 115 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, unsigned ExtendOpcode, 117 unsigned BitSetOpcode) const; 118 119 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I, Register SrcReg, 121 unsigned BitSetOpcode) const; 122 123 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I, Register SrcReg, 125 unsigned BitSetOpcode, bool SwapPrimarySide) const; 126 127 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I, Register SrcReg, 129 unsigned BitSetOpcode, 130 bool SwapPrimarySide) const; 131 132 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 133 const MachineInstr *Init = nullptr) const; 134 135 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I, std::vector<Register> SrcRegs, 137 unsigned Opcode) const; 138 139 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 140 unsigned Opcode) const; 141 142 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 143 MachineInstr &I) const; 144 145 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 bool selectStore(MachineInstr &I) const; 148 149 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 bool selectStackRestore(MachineInstr &I) const; 152 153 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 154 155 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I, unsigned NewOpcode, 157 unsigned NegateOpcode = 0) const; 158 159 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 162 bool selectFence(MachineInstr &I) const; 163 164 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I, unsigned OpType) const; 169 170 bool selectAll(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectAny(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I) const; 181 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 185 unsigned comparisonOpcode, MachineInstr &I) const; 186 bool selectCross(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 192 MachineInstr &I) const; 193 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 194 MachineInstr &I) const; 195 196 bool selectSign(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 199 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 200 MachineInstr &I) const; 201 202 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I, unsigned Opcode) const; 204 205 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I, bool Signed) const; 207 208 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 209 MachineInstr &I) const; 210 211 template <bool Signed> 212 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 template <bool Signed> 215 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 216 MachineInstr &I) const; 217 218 bool selectWaveReduceSum(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I) const; 220 221 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 222 int OpIdx) const; 223 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 224 int OpIdx) const; 225 226 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 227 MachineInstr &I) const; 228 229 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 230 bool IsSigned) const; 231 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 232 bool IsSigned, unsigned Opcode) const; 233 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 234 bool IsSigned) const; 235 236 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 237 MachineInstr &I) const; 238 239 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 240 bool IsSigned) const; 241 242 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 243 const SPIRVType *intTy, const SPIRVType *boolTy) const; 244 245 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 246 MachineInstr &I) const; 247 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 248 MachineInstr &I) const; 249 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 250 MachineInstr &I) const; 251 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I) const; 253 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 254 MachineInstr &I) const; 255 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 256 MachineInstr &I) const; 257 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I) const; 259 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 262 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 263 MachineInstr &I) const; 264 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 265 MachineInstr &I) const; 266 267 bool selectBranch(MachineInstr &I) const; 268 bool selectBranchCond(MachineInstr &I) const; 269 270 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 271 MachineInstr &I) const; 272 273 [[maybe_unused]] bool selectExtInst(Register ResVReg, 274 const SPIRVType *RestType, 275 MachineInstr &I, 276 GL::GLSLExtInst GLInst) const; 277 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 278 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 279 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 280 MachineInstr &I, CL::OpenCLExtInst CLInst, 281 GL::GLSLExtInst GLInst) const; 282 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 283 MachineInstr &I, const ExtInstList &ExtInsts) const; 284 285 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 286 MachineInstr &I) const; 287 288 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 289 MachineInstr &I) const; 290 291 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 292 MachineInstr &I, unsigned Opcode) const; 293 294 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 295 MachineInstr &I) const; 296 297 bool selectUnmergeValues(MachineInstr &I) const; 298 299 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 300 MachineInstr &I) const; 301 302 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 303 MachineInstr &I) const; 304 bool selectImageWriteIntrinsic(MachineInstr &I) const; 305 bool selectResourceGetPointer(Register &ResVReg, const SPIRVType *ResType, 306 MachineInstr &I) const; 307 308 // Utilities 309 std::pair<Register, bool> 310 buildI32Constant(uint32_t Val, MachineInstr &I, 311 const SPIRVType *ResType = nullptr) const; 312 313 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 314 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 315 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 316 MachineInstr &I) const; 317 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 318 319 bool wrapIntoSpecConstantOp(MachineInstr &I, 320 SmallVector<Register> &CompositeArgs) const; 321 322 Register getUcharPtrTypeReg(MachineInstr &I, 323 SPIRV::StorageClass::StorageClass SC) const; 324 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 325 Register Src, Register DestType, 326 uint32_t Opcode) const; 327 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 328 SPIRVType *SrcPtrTy) const; 329 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 330 uint32_t Binding, uint32_t ArraySize, 331 Register IndexReg, bool IsNonUniform, 332 MachineIRBuilder MIRBuilder) const; 333 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 334 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType, 335 Register &ReadReg, MachineInstr &InsertionPoint) const; 336 bool generateImageRead(Register &ResVReg, const SPIRVType *ResType, 337 Register ImageReg, Register IdxReg, DebugLoc Loc, 338 MachineInstr &Pos) const; 339 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 340 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 341 Register ResVReg, const SPIRVType *ResType, 342 MachineInstr &I) const; 343 bool loadHandleBeforePosition(Register &HandleReg, const SPIRVType *ResType, 344 GIntrinsic &HandleDef, MachineInstr &Pos) const; 345 }; 346 347 } // end anonymous namespace 348 349 #define GET_GLOBALISEL_IMPL 350 #include "SPIRVGenGlobalISel.inc" 351 #undef GET_GLOBALISEL_IMPL 352 353 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 354 const SPIRVSubtarget &ST, 355 const RegisterBankInfo &RBI) 356 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 357 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 358 #define GET_GLOBALISEL_PREDICATES_INIT 359 #include "SPIRVGenGlobalISel.inc" 360 #undef GET_GLOBALISEL_PREDICATES_INIT 361 #define GET_GLOBALISEL_TEMPORARIES_INIT 362 #include "SPIRVGenGlobalISel.inc" 363 #undef GET_GLOBALISEL_TEMPORARIES_INIT 364 { 365 } 366 367 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 368 CodeGenCoverage *CoverageInfo, 369 ProfileSummaryInfo *PSI, 370 BlockFrequencyInfo *BFI) { 371 MRI = &MF.getRegInfo(); 372 GR.setCurrentFunc(MF); 373 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 374 } 375 376 // Ensure that register classes correspond to pattern matching rules. 377 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 378 if (HasVRegsReset == &MF) 379 return; 380 HasVRegsReset = &MF; 381 382 MachineRegisterInfo &MRI = MF.getRegInfo(); 383 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 384 Register Reg = Register::index2VirtReg(I); 385 LLT RegType = MRI.getType(Reg); 386 if (RegType.isScalar()) 387 MRI.setType(Reg, LLT::scalar(64)); 388 else if (RegType.isPointer()) 389 MRI.setType(Reg, LLT::pointer(0, 64)); 390 else if (RegType.isVector()) 391 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 392 } 393 for (const auto &MBB : MF) { 394 for (const auto &MI : MBB) { 395 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 396 continue; 397 Register DstReg = MI.getOperand(0).getReg(); 398 LLT DstType = MRI.getType(DstReg); 399 Register SrcReg = MI.getOperand(1).getReg(); 400 LLT SrcType = MRI.getType(SrcReg); 401 if (DstType != SrcType) 402 MRI.setType(DstReg, MRI.getType(SrcReg)); 403 404 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 405 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 406 if (DstRC != SrcRC && SrcRC) 407 MRI.setRegClass(DstReg, SrcRC); 408 } 409 } 410 } 411 412 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 413 414 // Defined in SPIRVLegalizerInfo.cpp. 415 extern bool isTypeFoldingSupported(unsigned Opcode); 416 417 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 418 for (const auto &MO : MI.all_defs()) { 419 Register Reg = MO.getReg(); 420 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 421 return false; 422 } 423 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 424 MI.isLifetimeMarker()) 425 return false; 426 if (MI.isPHI()) 427 return true; 428 if (MI.mayStore() || MI.isCall() || 429 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 430 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 431 return false; 432 return true; 433 } 434 435 bool SPIRVInstructionSelector::select(MachineInstr &I) { 436 resetVRegsType(*I.getParent()->getParent()); 437 438 assert(I.getParent() && "Instruction should be in a basic block!"); 439 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 440 441 Register Opcode = I.getOpcode(); 442 // If it's not a GMIR instruction, we've selected it already. 443 if (!isPreISelGenericOpcode(Opcode)) { 444 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 445 Register DstReg = I.getOperand(0).getReg(); 446 Register SrcReg = I.getOperand(1).getReg(); 447 auto *Def = MRI->getVRegDef(SrcReg); 448 if (isTypeFoldingSupported(Def->getOpcode())) { 449 bool Res = selectImpl(I, *CoverageInfo); 450 LLVM_DEBUG({ 451 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 452 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 453 I.print(dbgs()); 454 } 455 }); 456 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 457 if (Res) { 458 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 459 DeadMIs.insert(Def); 460 return Res; 461 } 462 } 463 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 464 MRI->replaceRegWith(SrcReg, DstReg); 465 GR.invalidateMachineInstr(&I); 466 I.removeFromParent(); 467 return true; 468 } else if (I.getNumDefs() == 1) { 469 // Make all vregs 64 bits (for SPIR-V IDs). 470 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 471 } 472 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 473 } 474 475 if (DeadMIs.contains(&I)) { 476 // if the instruction has been already made dead by folding it away 477 // erase it 478 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 479 salvageDebugInfo(*MRI, I); 480 GR.invalidateMachineInstr(&I); 481 I.eraseFromParent(); 482 return true; 483 } 484 485 if (I.getNumOperands() != I.getNumExplicitOperands()) { 486 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 487 return false; 488 } 489 490 // Common code for getting return reg+type, and removing selected instr 491 // from parent occurs here. Instr-specific selection happens in spvSelect(). 492 bool HasDefs = I.getNumDefs() > 0; 493 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 494 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 495 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 496 if (spvSelect(ResVReg, ResType, I)) { 497 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 498 for (unsigned i = 0; i < I.getNumDefs(); ++i) 499 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 500 GR.invalidateMachineInstr(&I); 501 I.removeFromParent(); 502 return true; 503 } 504 return false; 505 } 506 507 static bool mayApplyGenericSelection(unsigned Opcode) { 508 switch (Opcode) { 509 case TargetOpcode::G_CONSTANT: 510 return false; 511 case TargetOpcode::G_SADDO: 512 case TargetOpcode::G_SSUBO: 513 return true; 514 } 515 return isTypeFoldingSupported(Opcode); 516 } 517 518 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 519 MachineInstr &I) const { 520 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 521 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 522 if (DstRC != SrcRC && SrcRC) 523 MRI->setRegClass(DestReg, SrcRC); 524 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 525 TII.get(TargetOpcode::COPY)) 526 .addDef(DestReg) 527 .addUse(SrcReg) 528 .constrainAllUses(TII, TRI, RBI); 529 } 530 531 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 532 const SPIRVType *ResType, 533 MachineInstr &I) const { 534 const unsigned Opcode = I.getOpcode(); 535 if (mayApplyGenericSelection(Opcode)) 536 return selectImpl(I, *CoverageInfo); 537 switch (Opcode) { 538 case TargetOpcode::G_CONSTANT: 539 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 540 I); 541 case TargetOpcode::G_GLOBAL_VALUE: 542 return selectGlobalValue(ResVReg, I); 543 case TargetOpcode::G_IMPLICIT_DEF: 544 return selectOpUndef(ResVReg, ResType, I); 545 case TargetOpcode::G_FREEZE: 546 return selectFreeze(ResVReg, ResType, I); 547 548 case TargetOpcode::G_INTRINSIC: 549 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 550 case TargetOpcode::G_INTRINSIC_CONVERGENT: 551 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 552 return selectIntrinsic(ResVReg, ResType, I); 553 case TargetOpcode::G_BITREVERSE: 554 return selectBitreverse(ResVReg, ResType, I); 555 556 case TargetOpcode::G_BUILD_VECTOR: 557 return selectBuildVector(ResVReg, ResType, I); 558 case TargetOpcode::G_SPLAT_VECTOR: 559 return selectSplatVector(ResVReg, ResType, I); 560 561 case TargetOpcode::G_SHUFFLE_VECTOR: { 562 MachineBasicBlock &BB = *I.getParent(); 563 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 564 .addDef(ResVReg) 565 .addUse(GR.getSPIRVTypeID(ResType)) 566 .addUse(I.getOperand(1).getReg()) 567 .addUse(I.getOperand(2).getReg()); 568 for (auto V : I.getOperand(3).getShuffleMask()) 569 MIB.addImm(V); 570 return MIB.constrainAllUses(TII, TRI, RBI); 571 } 572 case TargetOpcode::G_MEMMOVE: 573 case TargetOpcode::G_MEMCPY: 574 case TargetOpcode::G_MEMSET: 575 return selectMemOperation(ResVReg, I); 576 577 case TargetOpcode::G_ICMP: 578 return selectICmp(ResVReg, ResType, I); 579 case TargetOpcode::G_FCMP: 580 return selectFCmp(ResVReg, ResType, I); 581 582 case TargetOpcode::G_FRAME_INDEX: 583 return selectFrameIndex(ResVReg, ResType, I); 584 585 case TargetOpcode::G_LOAD: 586 return selectLoad(ResVReg, ResType, I); 587 case TargetOpcode::G_STORE: 588 return selectStore(I); 589 590 case TargetOpcode::G_BR: 591 return selectBranch(I); 592 case TargetOpcode::G_BRCOND: 593 return selectBranchCond(I); 594 595 case TargetOpcode::G_PHI: 596 return selectPhi(ResVReg, ResType, I); 597 598 case TargetOpcode::G_FPTOSI: 599 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 600 case TargetOpcode::G_FPTOUI: 601 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 602 603 case TargetOpcode::G_SITOFP: 604 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 605 case TargetOpcode::G_UITOFP: 606 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 607 608 case TargetOpcode::G_CTPOP: 609 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 610 case TargetOpcode::G_SMIN: 611 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 612 case TargetOpcode::G_UMIN: 613 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 614 615 case TargetOpcode::G_SMAX: 616 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 617 case TargetOpcode::G_UMAX: 618 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 619 620 case TargetOpcode::G_SCMP: 621 return selectSUCmp(ResVReg, ResType, I, true); 622 case TargetOpcode::G_UCMP: 623 return selectSUCmp(ResVReg, ResType, I, false); 624 625 case TargetOpcode::G_STRICT_FMA: 626 case TargetOpcode::G_FMA: 627 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 628 629 case TargetOpcode::G_STRICT_FLDEXP: 630 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 631 632 case TargetOpcode::G_FPOW: 633 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 634 case TargetOpcode::G_FPOWI: 635 return selectExtInst(ResVReg, ResType, I, CL::pown); 636 637 case TargetOpcode::G_FEXP: 638 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 639 case TargetOpcode::G_FEXP2: 640 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 641 642 case TargetOpcode::G_FLOG: 643 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 644 case TargetOpcode::G_FLOG2: 645 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 646 case TargetOpcode::G_FLOG10: 647 return selectLog10(ResVReg, ResType, I); 648 649 case TargetOpcode::G_FABS: 650 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 651 case TargetOpcode::G_ABS: 652 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 653 654 case TargetOpcode::G_FMINNUM: 655 case TargetOpcode::G_FMINIMUM: 656 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 657 case TargetOpcode::G_FMAXNUM: 658 case TargetOpcode::G_FMAXIMUM: 659 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 660 661 case TargetOpcode::G_FCOPYSIGN: 662 return selectExtInst(ResVReg, ResType, I, CL::copysign); 663 664 case TargetOpcode::G_FCEIL: 665 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 666 case TargetOpcode::G_FFLOOR: 667 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 668 669 case TargetOpcode::G_FCOS: 670 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 671 case TargetOpcode::G_FSIN: 672 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 673 case TargetOpcode::G_FTAN: 674 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 675 case TargetOpcode::G_FACOS: 676 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 677 case TargetOpcode::G_FASIN: 678 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 679 case TargetOpcode::G_FATAN: 680 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 681 case TargetOpcode::G_FATAN2: 682 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 683 case TargetOpcode::G_FCOSH: 684 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 685 case TargetOpcode::G_FSINH: 686 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 687 case TargetOpcode::G_FTANH: 688 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 689 690 case TargetOpcode::G_STRICT_FSQRT: 691 case TargetOpcode::G_FSQRT: 692 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 693 694 case TargetOpcode::G_CTTZ: 695 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 696 return selectExtInst(ResVReg, ResType, I, CL::ctz); 697 case TargetOpcode::G_CTLZ: 698 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 699 return selectExtInst(ResVReg, ResType, I, CL::clz); 700 701 case TargetOpcode::G_INTRINSIC_ROUND: 702 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 703 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 704 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 705 case TargetOpcode::G_INTRINSIC_TRUNC: 706 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 707 case TargetOpcode::G_FRINT: 708 case TargetOpcode::G_FNEARBYINT: 709 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 710 711 case TargetOpcode::G_SMULH: 712 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 713 case TargetOpcode::G_UMULH: 714 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 715 716 case TargetOpcode::G_SADDSAT: 717 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 718 case TargetOpcode::G_UADDSAT: 719 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 720 case TargetOpcode::G_SSUBSAT: 721 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 722 case TargetOpcode::G_USUBSAT: 723 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 724 725 case TargetOpcode::G_UADDO: 726 return selectOverflowArith(ResVReg, ResType, I, 727 ResType->getOpcode() == SPIRV::OpTypeVector 728 ? SPIRV::OpIAddCarryV 729 : SPIRV::OpIAddCarryS); 730 case TargetOpcode::G_USUBO: 731 return selectOverflowArith(ResVReg, ResType, I, 732 ResType->getOpcode() == SPIRV::OpTypeVector 733 ? SPIRV::OpISubBorrowV 734 : SPIRV::OpISubBorrowS); 735 case TargetOpcode::G_UMULO: 736 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 737 case TargetOpcode::G_SMULO: 738 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 739 740 case TargetOpcode::G_SEXT: 741 return selectExt(ResVReg, ResType, I, true); 742 case TargetOpcode::G_ANYEXT: 743 case TargetOpcode::G_ZEXT: 744 return selectExt(ResVReg, ResType, I, false); 745 case TargetOpcode::G_TRUNC: 746 return selectTrunc(ResVReg, ResType, I); 747 case TargetOpcode::G_FPTRUNC: 748 case TargetOpcode::G_FPEXT: 749 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 750 751 case TargetOpcode::G_PTRTOINT: 752 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 753 case TargetOpcode::G_INTTOPTR: 754 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 755 case TargetOpcode::G_BITCAST: 756 return selectBitcast(ResVReg, ResType, I); 757 case TargetOpcode::G_ADDRSPACE_CAST: 758 return selectAddrSpaceCast(ResVReg, ResType, I); 759 case TargetOpcode::G_PTR_ADD: { 760 // Currently, we get G_PTR_ADD only applied to global variables. 761 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 762 Register GV = I.getOperand(1).getReg(); 763 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 764 (void)II; 765 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 766 (*II).getOpcode() == TargetOpcode::COPY || 767 (*II).getOpcode() == SPIRV::OpVariable) && 768 isImm(I.getOperand(2), MRI)); 769 // It may be the initialization of a global variable. 770 bool IsGVInit = false; 771 for (MachineRegisterInfo::use_instr_iterator 772 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 773 UseEnd = MRI->use_instr_end(); 774 UseIt != UseEnd; UseIt = std::next(UseIt)) { 775 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 776 (*UseIt).getOpcode() == SPIRV::OpVariable) { 777 IsGVInit = true; 778 break; 779 } 780 } 781 MachineBasicBlock &BB = *I.getParent(); 782 if (!IsGVInit) { 783 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 784 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 785 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 786 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 787 // Build a new virtual register that is associated with the required 788 // data type. 789 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 790 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 791 // Having a correctly typed base we are ready to build the actually 792 // required GEP. It may not be a constant though, because all Operands 793 // of OpSpecConstantOp is to originate from other const instructions, 794 // and only the AccessChain named opcodes accept a global OpVariable 795 // instruction. We can't use an AccessChain opcode because of the type 796 // mismatch between result and base types. 797 if (!GR.isBitcastCompatible(ResType, GVType)) 798 report_fatal_error( 799 "incompatible result and operand types in a bitcast"); 800 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 801 MachineInstrBuilder MIB = 802 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 803 .addDef(NewVReg) 804 .addUse(ResTypeReg) 805 .addUse(GV); 806 return MIB.constrainAllUses(TII, TRI, RBI) && 807 BuildMI(BB, I, I.getDebugLoc(), 808 TII.get(STI.isVulkanEnv() 809 ? SPIRV::OpInBoundsAccessChain 810 : SPIRV::OpInBoundsPtrAccessChain)) 811 .addDef(ResVReg) 812 .addUse(ResTypeReg) 813 .addUse(NewVReg) 814 .addUse(I.getOperand(2).getReg()) 815 .constrainAllUses(TII, TRI, RBI); 816 } else { 817 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 818 .addDef(ResVReg) 819 .addUse(GR.getSPIRVTypeID(ResType)) 820 .addImm( 821 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 822 .addUse(GV) 823 .addUse(I.getOperand(2).getReg()) 824 .constrainAllUses(TII, TRI, RBI); 825 } 826 } 827 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 828 // initialize a global variable with a constant expression (e.g., the test 829 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 830 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 831 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 832 .addDef(ResVReg) 833 .addUse(GR.getSPIRVTypeID(ResType)) 834 .addImm(static_cast<uint32_t>( 835 SPIRV::Opcode::InBoundsPtrAccessChain)) 836 .addUse(GV) 837 .addUse(Idx) 838 .addUse(I.getOperand(2).getReg()); 839 return MIB.constrainAllUses(TII, TRI, RBI); 840 } 841 842 case TargetOpcode::G_ATOMICRMW_OR: 843 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 844 case TargetOpcode::G_ATOMICRMW_ADD: 845 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 846 case TargetOpcode::G_ATOMICRMW_AND: 847 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 848 case TargetOpcode::G_ATOMICRMW_MAX: 849 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 850 case TargetOpcode::G_ATOMICRMW_MIN: 851 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 852 case TargetOpcode::G_ATOMICRMW_SUB: 853 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 854 case TargetOpcode::G_ATOMICRMW_XOR: 855 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 856 case TargetOpcode::G_ATOMICRMW_UMAX: 857 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 858 case TargetOpcode::G_ATOMICRMW_UMIN: 859 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 860 case TargetOpcode::G_ATOMICRMW_XCHG: 861 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 862 case TargetOpcode::G_ATOMIC_CMPXCHG: 863 return selectAtomicCmpXchg(ResVReg, ResType, I); 864 865 case TargetOpcode::G_ATOMICRMW_FADD: 866 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 867 case TargetOpcode::G_ATOMICRMW_FSUB: 868 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 869 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 870 SPIRV::OpFNegate); 871 case TargetOpcode::G_ATOMICRMW_FMIN: 872 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 873 case TargetOpcode::G_ATOMICRMW_FMAX: 874 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 875 876 case TargetOpcode::G_FENCE: 877 return selectFence(I); 878 879 case TargetOpcode::G_STACKSAVE: 880 return selectStackSave(ResVReg, ResType, I); 881 case TargetOpcode::G_STACKRESTORE: 882 return selectStackRestore(I); 883 884 case TargetOpcode::G_UNMERGE_VALUES: 885 return selectUnmergeValues(I); 886 887 // Discard gen opcodes for intrinsics which we do not expect to actually 888 // represent code after lowering or intrinsics which are not implemented but 889 // should not crash when found in a customer's LLVM IR input. 890 case TargetOpcode::G_TRAP: 891 case TargetOpcode::G_DEBUGTRAP: 892 case TargetOpcode::G_UBSANTRAP: 893 case TargetOpcode::DBG_LABEL: 894 return true; 895 896 default: 897 return false; 898 } 899 } 900 901 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 902 const SPIRVType *ResType, 903 MachineInstr &I, 904 GL::GLSLExtInst GLInst) const { 905 return selectExtInst(ResVReg, ResType, I, 906 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 907 } 908 909 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 910 const SPIRVType *ResType, 911 MachineInstr &I, 912 CL::OpenCLExtInst CLInst) const { 913 return selectExtInst(ResVReg, ResType, I, 914 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 915 } 916 917 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 918 const SPIRVType *ResType, 919 MachineInstr &I, 920 CL::OpenCLExtInst CLInst, 921 GL::GLSLExtInst GLInst) const { 922 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 923 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 924 return selectExtInst(ResVReg, ResType, I, ExtInsts); 925 } 926 927 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 928 const SPIRVType *ResType, 929 MachineInstr &I, 930 const ExtInstList &Insts) const { 931 932 for (const auto &Ex : Insts) { 933 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 934 uint32_t Opcode = Ex.second; 935 if (STI.canUseExtInstSet(Set)) { 936 MachineBasicBlock &BB = *I.getParent(); 937 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 938 .addDef(ResVReg) 939 .addUse(GR.getSPIRVTypeID(ResType)) 940 .addImm(static_cast<uint32_t>(Set)) 941 .addImm(Opcode); 942 const unsigned NumOps = I.getNumOperands(); 943 unsigned Index = 1; 944 if (Index < NumOps && 945 I.getOperand(Index).getType() == 946 MachineOperand::MachineOperandType::MO_IntrinsicID) 947 Index = 2; 948 for (; Index < NumOps; ++Index) 949 MIB.add(I.getOperand(Index)); 950 return MIB.constrainAllUses(TII, TRI, RBI); 951 } 952 } 953 return false; 954 } 955 956 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 957 const SPIRVType *ResType, 958 MachineInstr &I, 959 std::vector<Register> Srcs, 960 unsigned Opcode) const { 961 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 962 .addDef(ResVReg) 963 .addUse(GR.getSPIRVTypeID(ResType)); 964 for (Register SReg : Srcs) { 965 MIB.addUse(SReg); 966 } 967 return MIB.constrainAllUses(TII, TRI, RBI); 968 } 969 970 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 971 const SPIRVType *ResType, 972 MachineInstr &I, 973 unsigned Opcode) const { 974 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 975 Register SrcReg = I.getOperand(1).getReg(); 976 bool IsGV = false; 977 for (MachineRegisterInfo::def_instr_iterator DefIt = 978 MRI->def_instr_begin(SrcReg); 979 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 980 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 981 IsGV = true; 982 break; 983 } 984 } 985 if (IsGV) { 986 uint32_t SpecOpcode = 0; 987 switch (Opcode) { 988 case SPIRV::OpConvertPtrToU: 989 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 990 break; 991 case SPIRV::OpConvertUToPtr: 992 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 993 break; 994 } 995 if (SpecOpcode) 996 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 997 TII.get(SPIRV::OpSpecConstantOp)) 998 .addDef(ResVReg) 999 .addUse(GR.getSPIRVTypeID(ResType)) 1000 .addImm(SpecOpcode) 1001 .addUse(SrcReg) 1002 .constrainAllUses(TII, TRI, RBI); 1003 } 1004 } 1005 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 1006 Opcode); 1007 } 1008 1009 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 1010 const SPIRVType *ResType, 1011 MachineInstr &I) const { 1012 Register OpReg = I.getOperand(1).getReg(); 1013 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 1014 if (!GR.isBitcastCompatible(ResType, OpType)) 1015 report_fatal_error("incompatible result and operand types in a bitcast"); 1016 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1017 } 1018 1019 static void addMemoryOperands(MachineMemOperand *MemOp, 1020 MachineInstrBuilder &MIB) { 1021 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1022 if (MemOp->isVolatile()) 1023 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1024 if (MemOp->isNonTemporal()) 1025 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1026 if (MemOp->getAlign().value()) 1027 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 1028 1029 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 1030 MIB.addImm(SpvMemOp); 1031 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 1032 MIB.addImm(MemOp->getAlign().value()); 1033 } 1034 } 1035 1036 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1037 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1038 if (Flags & MachineMemOperand::Flags::MOVolatile) 1039 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1040 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1041 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1042 1043 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1044 MIB.addImm(SpvMemOp); 1045 } 1046 1047 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1048 const SPIRVType *ResType, 1049 MachineInstr &I) const { 1050 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1051 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1052 1053 auto *PtrDef = getVRegDef(*MRI, Ptr); 1054 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); 1055 if (IntPtrDef && 1056 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 1057 Register ImageReg = IntPtrDef->getOperand(2).getReg(); 1058 Register NewImageReg = 1059 MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 1060 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 1061 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 1062 *ImageDef, I)) { 1063 return false; 1064 } 1065 1066 Register IdxReg = IntPtrDef->getOperand(3).getReg(); 1067 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, 1068 I.getDebugLoc(), I); 1069 } 1070 1071 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1072 .addDef(ResVReg) 1073 .addUse(GR.getSPIRVTypeID(ResType)) 1074 .addUse(Ptr); 1075 if (!I.getNumMemOperands()) { 1076 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1077 I.getOpcode() == 1078 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1079 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1080 } else { 1081 addMemoryOperands(*I.memoperands_begin(), MIB); 1082 } 1083 return MIB.constrainAllUses(TII, TRI, RBI); 1084 } 1085 1086 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1087 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1088 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1089 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1090 1091 auto *PtrDef = getVRegDef(*MRI, Ptr); 1092 auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); 1093 if (IntPtrDef && 1094 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 1095 Register ImageReg = IntPtrDef->getOperand(2).getReg(); 1096 Register NewImageReg = 1097 MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 1098 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 1099 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 1100 *ImageDef, I)) { 1101 return false; 1102 } 1103 1104 Register IdxReg = IntPtrDef->getOperand(3).getReg(); 1105 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1106 TII.get(SPIRV::OpImageWrite)) 1107 .addUse(NewImageReg) 1108 .addUse(IdxReg) 1109 .addUse(StoreVal) 1110 .constrainAllUses(TII, TRI, RBI); 1111 } 1112 1113 MachineBasicBlock &BB = *I.getParent(); 1114 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1115 .addUse(Ptr) 1116 .addUse(StoreVal); 1117 if (!I.getNumMemOperands()) { 1118 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1119 I.getOpcode() == 1120 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1121 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1122 } else { 1123 addMemoryOperands(*I.memoperands_begin(), MIB); 1124 } 1125 return MIB.constrainAllUses(TII, TRI, RBI); 1126 } 1127 1128 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1129 const SPIRVType *ResType, 1130 MachineInstr &I) const { 1131 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1132 report_fatal_error( 1133 "llvm.stacksave intrinsic: this instruction requires the following " 1134 "SPIR-V extension: SPV_INTEL_variable_length_array", 1135 false); 1136 MachineBasicBlock &BB = *I.getParent(); 1137 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1138 .addDef(ResVReg) 1139 .addUse(GR.getSPIRVTypeID(ResType)) 1140 .constrainAllUses(TII, TRI, RBI); 1141 } 1142 1143 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1144 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1145 report_fatal_error( 1146 "llvm.stackrestore intrinsic: this instruction requires the following " 1147 "SPIR-V extension: SPV_INTEL_variable_length_array", 1148 false); 1149 if (!I.getOperand(0).isReg()) 1150 return false; 1151 MachineBasicBlock &BB = *I.getParent(); 1152 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1153 .addUse(I.getOperand(0).getReg()) 1154 .constrainAllUses(TII, TRI, RBI); 1155 } 1156 1157 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1158 MachineInstr &I) const { 1159 MachineBasicBlock &BB = *I.getParent(); 1160 Register SrcReg = I.getOperand(1).getReg(); 1161 bool Result = true; 1162 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1163 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1164 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1165 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1166 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1167 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1168 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1169 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1170 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1171 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1172 Function &CurFunction = GR.CurMF->getFunction(); 1173 Type *LLVMArrTy = 1174 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1175 // Module takes ownership of the global var. 1176 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1177 true, GlobalValue::InternalLinkage, 1178 Constant::getNullValue(LLVMArrTy)); 1179 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1180 GR.add(GV, GR.CurMF, VarReg); 1181 GR.addGlobalObject(GV, GR.CurMF, VarReg); 1182 1183 Result &= 1184 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1185 .addDef(VarReg) 1186 .addUse(GR.getSPIRVTypeID(VarTy)) 1187 .addImm(SPIRV::StorageClass::UniformConstant) 1188 .addUse(Const) 1189 .constrainAllUses(TII, TRI, RBI); 1190 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1191 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1192 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1193 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1194 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1195 } 1196 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1197 .addUse(I.getOperand(0).getReg()) 1198 .addUse(SrcReg) 1199 .addUse(I.getOperand(2).getReg()); 1200 if (I.getNumMemOperands()) 1201 addMemoryOperands(*I.memoperands_begin(), MIB); 1202 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1203 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1204 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1205 return Result; 1206 } 1207 1208 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1209 const SPIRVType *ResType, 1210 MachineInstr &I, 1211 unsigned NewOpcode, 1212 unsigned NegateOpcode) const { 1213 bool Result = true; 1214 assert(I.hasOneMemOperand()); 1215 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1216 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1217 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1218 auto ScopeConstant = buildI32Constant(Scope, I); 1219 Register ScopeReg = ScopeConstant.first; 1220 Result &= ScopeConstant.second; 1221 1222 Register Ptr = I.getOperand(1).getReg(); 1223 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1224 // auto ScSem = 1225 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1226 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1227 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1228 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1229 Register MemSemReg = MemSemConstant.first; 1230 Result &= MemSemConstant.second; 1231 1232 Register ValueReg = I.getOperand(2).getReg(); 1233 if (NegateOpcode != 0) { 1234 // Translation with negative value operand is requested 1235 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1236 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1237 ValueReg = TmpReg; 1238 } 1239 1240 return Result && 1241 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1242 .addDef(ResVReg) 1243 .addUse(GR.getSPIRVTypeID(ResType)) 1244 .addUse(Ptr) 1245 .addUse(ScopeReg) 1246 .addUse(MemSemReg) 1247 .addUse(ValueReg) 1248 .constrainAllUses(TII, TRI, RBI); 1249 } 1250 1251 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1252 unsigned ArgI = I.getNumOperands() - 1; 1253 Register SrcReg = 1254 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1255 SPIRVType *DefType = 1256 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1257 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1258 report_fatal_error( 1259 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1260 1261 SPIRVType *ScalarType = 1262 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1263 MachineBasicBlock &BB = *I.getParent(); 1264 bool Res = false; 1265 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1266 Register ResVReg = I.getOperand(i).getReg(); 1267 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1268 if (!ResType) { 1269 // There was no "assign type" actions, let's fix this now 1270 ResType = ScalarType; 1271 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1272 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1273 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1274 } 1275 auto MIB = 1276 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1277 .addDef(ResVReg) 1278 .addUse(GR.getSPIRVTypeID(ResType)) 1279 .addUse(SrcReg) 1280 .addImm(static_cast<int64_t>(i)); 1281 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1282 } 1283 return Res; 1284 } 1285 1286 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1287 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1288 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1289 auto MemSemConstant = buildI32Constant(MemSem, I); 1290 Register MemSemReg = MemSemConstant.first; 1291 bool Result = MemSemConstant.second; 1292 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1293 uint32_t Scope = static_cast<uint32_t>( 1294 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1295 auto ScopeConstant = buildI32Constant(Scope, I); 1296 Register ScopeReg = ScopeConstant.first; 1297 Result &= ScopeConstant.second; 1298 MachineBasicBlock &BB = *I.getParent(); 1299 return Result && 1300 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1301 .addUse(ScopeReg) 1302 .addUse(MemSemReg) 1303 .constrainAllUses(TII, TRI, RBI); 1304 } 1305 1306 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1307 const SPIRVType *ResType, 1308 MachineInstr &I, 1309 unsigned Opcode) const { 1310 Type *ResTy = nullptr; 1311 StringRef ResName; 1312 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1313 report_fatal_error( 1314 "Not enough info to select the arithmetic with overflow instruction"); 1315 if (!ResTy || !ResTy->isStructTy()) 1316 report_fatal_error("Expect struct type result for the arithmetic " 1317 "with overflow instruction"); 1318 // "Result Type must be from OpTypeStruct. The struct must have two members, 1319 // and the two members must be the same type." 1320 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1321 ResTy = StructType::get(ResElemTy, ResElemTy); 1322 // Build SPIR-V types and constant(s) if needed. 1323 MachineIRBuilder MIRBuilder(I); 1324 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1325 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1326 assert(I.getNumDefs() > 1 && "Not enought operands"); 1327 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1328 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1329 if (N > 1) 1330 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1331 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1332 Register ZeroReg = buildZerosVal(ResType, I); 1333 // A new virtual register to store the result struct. 1334 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1335 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1336 // Build the result name if needed. 1337 if (ResName.size() > 0) 1338 buildOpName(StructVReg, ResName, MIRBuilder); 1339 // Build the arithmetic with overflow instruction. 1340 MachineBasicBlock &BB = *I.getParent(); 1341 auto MIB = 1342 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1343 .addDef(StructVReg) 1344 .addUse(GR.getSPIRVTypeID(StructType)); 1345 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1346 MIB.addUse(I.getOperand(i).getReg()); 1347 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1348 // Build instructions to extract fields of the instruction's result. 1349 // A new virtual register to store the higher part of the result struct. 1350 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1351 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1352 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1353 auto MIB = 1354 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1355 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1356 .addUse(GR.getSPIRVTypeID(ResType)) 1357 .addUse(StructVReg) 1358 .addImm(i); 1359 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1360 } 1361 // Build boolean value from the higher part. 1362 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1363 .addDef(I.getOperand(1).getReg()) 1364 .addUse(BoolTypeReg) 1365 .addUse(HigherVReg) 1366 .addUse(ZeroReg) 1367 .constrainAllUses(TII, TRI, RBI); 1368 } 1369 1370 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1371 const SPIRVType *ResType, 1372 MachineInstr &I) const { 1373 bool Result = true; 1374 Register ScopeReg; 1375 Register MemSemEqReg; 1376 Register MemSemNeqReg; 1377 Register Ptr = I.getOperand(2).getReg(); 1378 if (!isa<GIntrinsic>(I)) { 1379 assert(I.hasOneMemOperand()); 1380 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1381 unsigned Scope = static_cast<uint32_t>(getMemScope( 1382 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1383 auto ScopeConstant = buildI32Constant(Scope, I); 1384 ScopeReg = ScopeConstant.first; 1385 Result &= ScopeConstant.second; 1386 1387 unsigned ScSem = static_cast<uint32_t>( 1388 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1389 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1390 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1391 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1392 MemSemEqReg = MemSemEqConstant.first; 1393 Result &= MemSemEqConstant.second; 1394 AtomicOrdering FO = MemOp->getFailureOrdering(); 1395 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1396 if (MemSemEq == MemSemNeq) 1397 MemSemNeqReg = MemSemEqReg; 1398 else { 1399 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1400 MemSemNeqReg = MemSemNeqConstant.first; 1401 Result &= MemSemNeqConstant.second; 1402 } 1403 } else { 1404 ScopeReg = I.getOperand(5).getReg(); 1405 MemSemEqReg = I.getOperand(6).getReg(); 1406 MemSemNeqReg = I.getOperand(7).getReg(); 1407 } 1408 1409 Register Cmp = I.getOperand(3).getReg(); 1410 Register Val = I.getOperand(4).getReg(); 1411 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1412 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1413 const DebugLoc &DL = I.getDebugLoc(); 1414 Result &= 1415 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1416 .addDef(ACmpRes) 1417 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1418 .addUse(Ptr) 1419 .addUse(ScopeReg) 1420 .addUse(MemSemEqReg) 1421 .addUse(MemSemNeqReg) 1422 .addUse(Val) 1423 .addUse(Cmp) 1424 .constrainAllUses(TII, TRI, RBI); 1425 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1426 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1427 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1428 .addDef(CmpSuccReg) 1429 .addUse(GR.getSPIRVTypeID(BoolTy)) 1430 .addUse(ACmpRes) 1431 .addUse(Cmp) 1432 .constrainAllUses(TII, TRI, RBI); 1433 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1434 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1435 .addDef(TmpReg) 1436 .addUse(GR.getSPIRVTypeID(ResType)) 1437 .addUse(ACmpRes) 1438 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1439 .addImm(0) 1440 .constrainAllUses(TII, TRI, RBI); 1441 return Result && 1442 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1443 .addDef(ResVReg) 1444 .addUse(GR.getSPIRVTypeID(ResType)) 1445 .addUse(CmpSuccReg) 1446 .addUse(TmpReg) 1447 .addImm(1) 1448 .constrainAllUses(TII, TRI, RBI); 1449 } 1450 1451 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1452 switch (SC) { 1453 case SPIRV::StorageClass::Workgroup: 1454 case SPIRV::StorageClass::CrossWorkgroup: 1455 case SPIRV::StorageClass::Function: 1456 return true; 1457 default: 1458 return false; 1459 } 1460 } 1461 1462 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1463 switch (SC) { 1464 case SPIRV::StorageClass::DeviceOnlyINTEL: 1465 case SPIRV::StorageClass::HostOnlyINTEL: 1466 return true; 1467 default: 1468 return false; 1469 } 1470 } 1471 1472 // Returns true ResVReg is referred only from global vars and OpName's. 1473 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1474 bool IsGRef = false; 1475 bool IsAllowedRefs = 1476 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1477 [&IsGRef](auto const &It) { 1478 unsigned Opcode = It.getOpcode(); 1479 if (Opcode == SPIRV::OpConstantComposite || 1480 Opcode == SPIRV::OpVariable || 1481 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1482 return IsGRef = true; 1483 return Opcode == SPIRV::OpName; 1484 }); 1485 return IsAllowedRefs && IsGRef; 1486 } 1487 1488 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1489 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1490 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1491 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1492 } 1493 1494 MachineInstrBuilder 1495 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1496 Register Src, Register DestType, 1497 uint32_t Opcode) const { 1498 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1499 TII.get(SPIRV::OpSpecConstantOp)) 1500 .addDef(Dest) 1501 .addUse(DestType) 1502 .addImm(Opcode) 1503 .addUse(Src); 1504 } 1505 1506 MachineInstrBuilder 1507 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1508 SPIRVType *SrcPtrTy) const { 1509 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1510 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1511 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1512 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1513 SPIRV::StorageClass::Generic), 1514 GR.getPointerSize())); 1515 MachineFunction *MF = I.getParent()->getParent(); 1516 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1517 MachineInstrBuilder MIB = buildSpecConstantOp( 1518 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1519 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1520 GR.add(MIB.getInstr(), MF, Tmp); 1521 return MIB; 1522 } 1523 1524 // In SPIR-V address space casting can only happen to and from the Generic 1525 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1526 // pointers to and from Generic pointers. As such, we can convert e.g. from 1527 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1528 // other combinations can only be done by a bitcast, and are probably not safe. 1529 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1530 const SPIRVType *ResType, 1531 MachineInstr &I) const { 1532 MachineBasicBlock &BB = *I.getParent(); 1533 const DebugLoc &DL = I.getDebugLoc(); 1534 1535 Register SrcPtr = I.getOperand(1).getReg(); 1536 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1537 1538 // don't generate a cast for a null that may be represented by OpTypeInt 1539 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1540 ResType->getOpcode() != SPIRV::OpTypePointer) 1541 return BuildCOPY(ResVReg, SrcPtr, I); 1542 1543 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1544 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1545 1546 if (isASCastInGVar(MRI, ResVReg)) { 1547 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1548 // are expressed by OpSpecConstantOp with an Opcode. 1549 // TODO: maybe insert a check whether the Kernel capability was declared and 1550 // so PtrCastToGeneric/GenericCastToPtr are available. 1551 unsigned SpecOpcode = 1552 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1553 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1554 : (SrcSC == SPIRV::StorageClass::Generic && 1555 isGenericCastablePtr(DstSC) 1556 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1557 : 0); 1558 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1559 // correct value of ResType and use general i8* instead. Maybe this should 1560 // be addressed in the emit-intrinsic step to infer a correct 1561 // OpConstantComposite type. 1562 if (SpecOpcode) { 1563 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1564 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1565 .constrainAllUses(TII, TRI, RBI); 1566 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1567 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1568 return MIB.constrainAllUses(TII, TRI, RBI) && 1569 buildSpecConstantOp( 1570 I, ResVReg, MIB->getOperand(0).getReg(), 1571 getUcharPtrTypeReg(I, DstSC), 1572 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1573 .constrainAllUses(TII, TRI, RBI); 1574 } 1575 } 1576 1577 // don't generate a cast between identical storage classes 1578 if (SrcSC == DstSC) 1579 return BuildCOPY(ResVReg, SrcPtr, I); 1580 1581 if ((SrcSC == SPIRV::StorageClass::Function && 1582 DstSC == SPIRV::StorageClass::Private) || 1583 (DstSC == SPIRV::StorageClass::Function && 1584 SrcSC == SPIRV::StorageClass::Private)) 1585 return BuildCOPY(ResVReg, SrcPtr, I); 1586 1587 // Casting from an eligible pointer to Generic. 1588 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1589 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1590 // Casting from Generic to an eligible pointer. 1591 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1592 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1593 // Casting between 2 eligible pointers using Generic as an intermediary. 1594 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1595 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1596 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1597 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1598 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1599 .addDef(Tmp) 1600 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1601 .addUse(SrcPtr) 1602 .constrainAllUses(TII, TRI, RBI); 1603 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1604 .addDef(ResVReg) 1605 .addUse(GR.getSPIRVTypeID(ResType)) 1606 .addUse(Tmp) 1607 .constrainAllUses(TII, TRI, RBI); 1608 } 1609 1610 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1611 // be applied 1612 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1613 return selectUnOp(ResVReg, ResType, I, 1614 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1615 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1616 return selectUnOp(ResVReg, ResType, I, 1617 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1618 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1619 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1620 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1621 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1622 1623 // Bitcast for pointers requires that the address spaces must match 1624 return false; 1625 } 1626 1627 static unsigned getFCmpOpcode(unsigned PredNum) { 1628 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1629 switch (Pred) { 1630 case CmpInst::FCMP_OEQ: 1631 return SPIRV::OpFOrdEqual; 1632 case CmpInst::FCMP_OGE: 1633 return SPIRV::OpFOrdGreaterThanEqual; 1634 case CmpInst::FCMP_OGT: 1635 return SPIRV::OpFOrdGreaterThan; 1636 case CmpInst::FCMP_OLE: 1637 return SPIRV::OpFOrdLessThanEqual; 1638 case CmpInst::FCMP_OLT: 1639 return SPIRV::OpFOrdLessThan; 1640 case CmpInst::FCMP_ONE: 1641 return SPIRV::OpFOrdNotEqual; 1642 case CmpInst::FCMP_ORD: 1643 return SPIRV::OpOrdered; 1644 case CmpInst::FCMP_UEQ: 1645 return SPIRV::OpFUnordEqual; 1646 case CmpInst::FCMP_UGE: 1647 return SPIRV::OpFUnordGreaterThanEqual; 1648 case CmpInst::FCMP_UGT: 1649 return SPIRV::OpFUnordGreaterThan; 1650 case CmpInst::FCMP_ULE: 1651 return SPIRV::OpFUnordLessThanEqual; 1652 case CmpInst::FCMP_ULT: 1653 return SPIRV::OpFUnordLessThan; 1654 case CmpInst::FCMP_UNE: 1655 return SPIRV::OpFUnordNotEqual; 1656 case CmpInst::FCMP_UNO: 1657 return SPIRV::OpUnordered; 1658 default: 1659 llvm_unreachable("Unknown predicate type for FCmp"); 1660 } 1661 } 1662 1663 static unsigned getICmpOpcode(unsigned PredNum) { 1664 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1665 switch (Pred) { 1666 case CmpInst::ICMP_EQ: 1667 return SPIRV::OpIEqual; 1668 case CmpInst::ICMP_NE: 1669 return SPIRV::OpINotEqual; 1670 case CmpInst::ICMP_SGE: 1671 return SPIRV::OpSGreaterThanEqual; 1672 case CmpInst::ICMP_SGT: 1673 return SPIRV::OpSGreaterThan; 1674 case CmpInst::ICMP_SLE: 1675 return SPIRV::OpSLessThanEqual; 1676 case CmpInst::ICMP_SLT: 1677 return SPIRV::OpSLessThan; 1678 case CmpInst::ICMP_UGE: 1679 return SPIRV::OpUGreaterThanEqual; 1680 case CmpInst::ICMP_UGT: 1681 return SPIRV::OpUGreaterThan; 1682 case CmpInst::ICMP_ULE: 1683 return SPIRV::OpULessThanEqual; 1684 case CmpInst::ICMP_ULT: 1685 return SPIRV::OpULessThan; 1686 default: 1687 llvm_unreachable("Unknown predicate type for ICmp"); 1688 } 1689 } 1690 1691 static unsigned getPtrCmpOpcode(unsigned Pred) { 1692 switch (static_cast<CmpInst::Predicate>(Pred)) { 1693 case CmpInst::ICMP_EQ: 1694 return SPIRV::OpPtrEqual; 1695 case CmpInst::ICMP_NE: 1696 return SPIRV::OpPtrNotEqual; 1697 default: 1698 llvm_unreachable("Unknown predicate type for pointer comparison"); 1699 } 1700 } 1701 1702 // Return the logical operation, or abort if none exists. 1703 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1704 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1705 switch (Pred) { 1706 case CmpInst::ICMP_EQ: 1707 return SPIRV::OpLogicalEqual; 1708 case CmpInst::ICMP_NE: 1709 return SPIRV::OpLogicalNotEqual; 1710 default: 1711 llvm_unreachable("Unknown predicate type for Bool comparison"); 1712 } 1713 } 1714 1715 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1716 if (!LLVMFloatTy) 1717 return APFloat::getZero(APFloat::IEEEsingle()); 1718 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1719 case Type::HalfTyID: 1720 return APFloat::getZero(APFloat::IEEEhalf()); 1721 default: 1722 case Type::FloatTyID: 1723 return APFloat::getZero(APFloat::IEEEsingle()); 1724 case Type::DoubleTyID: 1725 return APFloat::getZero(APFloat::IEEEdouble()); 1726 } 1727 } 1728 1729 static APFloat getOneFP(const Type *LLVMFloatTy) { 1730 if (!LLVMFloatTy) 1731 return APFloat::getOne(APFloat::IEEEsingle()); 1732 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1733 case Type::HalfTyID: 1734 return APFloat::getOne(APFloat::IEEEhalf()); 1735 default: 1736 case Type::FloatTyID: 1737 return APFloat::getOne(APFloat::IEEEsingle()); 1738 case Type::DoubleTyID: 1739 return APFloat::getOne(APFloat::IEEEdouble()); 1740 } 1741 } 1742 1743 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1744 const SPIRVType *ResType, 1745 MachineInstr &I, 1746 unsigned OpAnyOrAll) const { 1747 assert(I.getNumOperands() == 3); 1748 assert(I.getOperand(2).isReg()); 1749 MachineBasicBlock &BB = *I.getParent(); 1750 Register InputRegister = I.getOperand(2).getReg(); 1751 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1752 1753 if (!InputType) 1754 report_fatal_error("Input Type could not be determined."); 1755 1756 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1757 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1758 if (IsBoolTy && !IsVectorTy) { 1759 assert(ResVReg == I.getOperand(0).getReg()); 1760 return BuildCOPY(ResVReg, InputRegister, I); 1761 } 1762 1763 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1764 unsigned SpirvNotEqualId = 1765 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1766 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1767 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1768 Register NotEqualReg = ResVReg; 1769 1770 if (IsVectorTy) { 1771 NotEqualReg = IsBoolTy ? InputRegister 1772 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1773 const unsigned NumElts = InputType->getOperand(2).getImm(); 1774 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1775 } 1776 1777 bool Result = true; 1778 if (!IsBoolTy) { 1779 Register ConstZeroReg = 1780 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1781 1782 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1783 .addDef(NotEqualReg) 1784 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1785 .addUse(InputRegister) 1786 .addUse(ConstZeroReg) 1787 .constrainAllUses(TII, TRI, RBI); 1788 } 1789 1790 if (!IsVectorTy) 1791 return Result; 1792 1793 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1794 .addDef(ResVReg) 1795 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1796 .addUse(NotEqualReg) 1797 .constrainAllUses(TII, TRI, RBI); 1798 } 1799 1800 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1801 const SPIRVType *ResType, 1802 MachineInstr &I) const { 1803 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1804 } 1805 1806 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1807 const SPIRVType *ResType, 1808 MachineInstr &I) const { 1809 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1810 } 1811 1812 // Select the OpDot instruction for the given float dot 1813 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1814 const SPIRVType *ResType, 1815 MachineInstr &I) const { 1816 assert(I.getNumOperands() == 4); 1817 assert(I.getOperand(2).isReg()); 1818 assert(I.getOperand(3).isReg()); 1819 1820 [[maybe_unused]] SPIRVType *VecType = 1821 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1822 1823 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1824 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1825 "dot product requires a vector of at least 2 components"); 1826 1827 [[maybe_unused]] SPIRVType *EltType = 1828 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1829 1830 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1831 1832 MachineBasicBlock &BB = *I.getParent(); 1833 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1834 .addDef(ResVReg) 1835 .addUse(GR.getSPIRVTypeID(ResType)) 1836 .addUse(I.getOperand(2).getReg()) 1837 .addUse(I.getOperand(3).getReg()) 1838 .constrainAllUses(TII, TRI, RBI); 1839 } 1840 1841 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1842 const SPIRVType *ResType, 1843 MachineInstr &I, 1844 bool Signed) const { 1845 assert(I.getNumOperands() == 4); 1846 assert(I.getOperand(2).isReg()); 1847 assert(I.getOperand(3).isReg()); 1848 MachineBasicBlock &BB = *I.getParent(); 1849 1850 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1851 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1852 .addDef(ResVReg) 1853 .addUse(GR.getSPIRVTypeID(ResType)) 1854 .addUse(I.getOperand(2).getReg()) 1855 .addUse(I.getOperand(3).getReg()) 1856 .constrainAllUses(TII, TRI, RBI); 1857 } 1858 1859 // Since pre-1.6 SPIRV has no integer dot implementation, 1860 // expand by piecewise multiplying and adding the results 1861 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1862 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1863 assert(I.getNumOperands() == 4); 1864 assert(I.getOperand(2).isReg()); 1865 assert(I.getOperand(3).isReg()); 1866 MachineBasicBlock &BB = *I.getParent(); 1867 1868 // Multiply the vectors, then sum the results 1869 Register Vec0 = I.getOperand(2).getReg(); 1870 Register Vec1 = I.getOperand(3).getReg(); 1871 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1872 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1873 1874 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1875 .addDef(TmpVec) 1876 .addUse(GR.getSPIRVTypeID(VecType)) 1877 .addUse(Vec0) 1878 .addUse(Vec1) 1879 .constrainAllUses(TII, TRI, RBI); 1880 1881 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1882 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1883 "dot product requires a vector of at least 2 components"); 1884 1885 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1886 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1887 .addDef(Res) 1888 .addUse(GR.getSPIRVTypeID(ResType)) 1889 .addUse(TmpVec) 1890 .addImm(0) 1891 .constrainAllUses(TII, TRI, RBI); 1892 1893 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1894 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1895 1896 Result &= 1897 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1898 .addDef(Elt) 1899 .addUse(GR.getSPIRVTypeID(ResType)) 1900 .addUse(TmpVec) 1901 .addImm(i) 1902 .constrainAllUses(TII, TRI, RBI); 1903 1904 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1905 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1906 : ResVReg; 1907 1908 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1909 .addDef(Sum) 1910 .addUse(GR.getSPIRVTypeID(ResType)) 1911 .addUse(Res) 1912 .addUse(Elt) 1913 .constrainAllUses(TII, TRI, RBI); 1914 Res = Sum; 1915 } 1916 1917 return Result; 1918 } 1919 1920 template <bool Signed> 1921 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1922 const SPIRVType *ResType, 1923 MachineInstr &I) const { 1924 assert(I.getNumOperands() == 5); 1925 assert(I.getOperand(2).isReg()); 1926 assert(I.getOperand(3).isReg()); 1927 assert(I.getOperand(4).isReg()); 1928 MachineBasicBlock &BB = *I.getParent(); 1929 1930 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1931 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1932 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1933 .addDef(Dot) 1934 .addUse(GR.getSPIRVTypeID(ResType)) 1935 .addUse(I.getOperand(2).getReg()) 1936 .addUse(I.getOperand(3).getReg()) 1937 .constrainAllUses(TII, TRI, RBI); 1938 1939 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1940 .addDef(ResVReg) 1941 .addUse(GR.getSPIRVTypeID(ResType)) 1942 .addUse(Dot) 1943 .addUse(I.getOperand(4).getReg()) 1944 .constrainAllUses(TII, TRI, RBI); 1945 } 1946 1947 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1948 // extract the elements of the packed inputs, multiply them and add the result 1949 // to the accumulator. 1950 template <bool Signed> 1951 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1952 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1953 assert(I.getNumOperands() == 5); 1954 assert(I.getOperand(2).isReg()); 1955 assert(I.getOperand(3).isReg()); 1956 assert(I.getOperand(4).isReg()); 1957 MachineBasicBlock &BB = *I.getParent(); 1958 1959 bool Result = true; 1960 1961 // Acc = C 1962 Register Acc = I.getOperand(4).getReg(); 1963 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1964 auto ExtractOp = 1965 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1966 1967 // Extract the i8 element, multiply and add it to the accumulator 1968 for (unsigned i = 0; i < 4; i++) { 1969 // A[i] 1970 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1971 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1972 .addDef(AElt) 1973 .addUse(GR.getSPIRVTypeID(ResType)) 1974 .addUse(I.getOperand(2).getReg()) 1975 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1976 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1977 .constrainAllUses(TII, TRI, RBI); 1978 1979 // B[i] 1980 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1981 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1982 .addDef(BElt) 1983 .addUse(GR.getSPIRVTypeID(ResType)) 1984 .addUse(I.getOperand(3).getReg()) 1985 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1986 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1987 .constrainAllUses(TII, TRI, RBI); 1988 1989 // A[i] * B[i] 1990 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1991 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1992 .addDef(Mul) 1993 .addUse(GR.getSPIRVTypeID(ResType)) 1994 .addUse(AElt) 1995 .addUse(BElt) 1996 .constrainAllUses(TII, TRI, RBI); 1997 1998 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1999 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2000 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 2001 .addDef(MaskMul) 2002 .addUse(GR.getSPIRVTypeID(ResType)) 2003 .addUse(Mul) 2004 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 2005 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 2006 .constrainAllUses(TII, TRI, RBI); 2007 2008 // Acc = Acc + A[i] * B[i] 2009 Register Sum = 2010 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 2011 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 2012 .addDef(Sum) 2013 .addUse(GR.getSPIRVTypeID(ResType)) 2014 .addUse(Acc) 2015 .addUse(MaskMul) 2016 .constrainAllUses(TII, TRI, RBI); 2017 2018 Acc = Sum; 2019 } 2020 2021 return Result; 2022 } 2023 2024 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 2025 /// does not have a saturate builtin. 2026 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 2027 const SPIRVType *ResType, 2028 MachineInstr &I) const { 2029 assert(I.getNumOperands() == 3); 2030 assert(I.getOperand(2).isReg()); 2031 MachineBasicBlock &BB = *I.getParent(); 2032 Register VZero = buildZerosValF(ResType, I); 2033 Register VOne = buildOnesValF(ResType, I); 2034 2035 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2036 .addDef(ResVReg) 2037 .addUse(GR.getSPIRVTypeID(ResType)) 2038 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2039 .addImm(GL::FClamp) 2040 .addUse(I.getOperand(2).getReg()) 2041 .addUse(VZero) 2042 .addUse(VOne) 2043 .constrainAllUses(TII, TRI, RBI); 2044 } 2045 2046 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 2047 const SPIRVType *ResType, 2048 MachineInstr &I) const { 2049 assert(I.getNumOperands() == 3); 2050 assert(I.getOperand(2).isReg()); 2051 MachineBasicBlock &BB = *I.getParent(); 2052 Register InputRegister = I.getOperand(2).getReg(); 2053 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2054 auto &DL = I.getDebugLoc(); 2055 2056 if (!InputType) 2057 report_fatal_error("Input Type could not be determined."); 2058 2059 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2060 2061 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 2062 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 2063 2064 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 2065 2066 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 2067 Register SignReg = NeedsConversion 2068 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 2069 : ResVReg; 2070 2071 bool Result = 2072 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 2073 .addDef(SignReg) 2074 .addUse(GR.getSPIRVTypeID(InputType)) 2075 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2076 .addImm(SignOpcode) 2077 .addUse(InputRegister) 2078 .constrainAllUses(TII, TRI, RBI); 2079 2080 if (NeedsConversion) { 2081 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2082 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2083 .addDef(ResVReg) 2084 .addUse(GR.getSPIRVTypeID(ResType)) 2085 .addUse(SignReg) 2086 .constrainAllUses(TII, TRI, RBI); 2087 } 2088 2089 return Result; 2090 } 2091 2092 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2093 const SPIRVType *ResType, 2094 MachineInstr &I, 2095 unsigned Opcode) const { 2096 MachineBasicBlock &BB = *I.getParent(); 2097 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2098 2099 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2100 .addDef(ResVReg) 2101 .addUse(GR.getSPIRVTypeID(ResType)) 2102 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2103 IntTy, TII)); 2104 2105 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2106 BMI.addUse(I.getOperand(J).getReg()); 2107 } 2108 2109 return BMI.constrainAllUses(TII, TRI, RBI); 2110 } 2111 2112 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2113 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2114 2115 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2116 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2117 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2118 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2119 SPIRV::OpGroupNonUniformBallot); 2120 2121 MachineBasicBlock &BB = *I.getParent(); 2122 Result &= 2123 BuildMI(BB, I, I.getDebugLoc(), 2124 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2125 .addDef(ResVReg) 2126 .addUse(GR.getSPIRVTypeID(ResType)) 2127 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2128 .addImm(SPIRV::GroupOperation::Reduce) 2129 .addUse(BallotReg) 2130 .constrainAllUses(TII, TRI, RBI); 2131 2132 return Result; 2133 } 2134 2135 bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg, 2136 const SPIRVType *ResType, 2137 MachineInstr &I) const { 2138 assert(I.getNumOperands() == 3); 2139 assert(I.getOperand(2).isReg()); 2140 MachineBasicBlock &BB = *I.getParent(); 2141 Register InputRegister = I.getOperand(2).getReg(); 2142 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2143 2144 if (!InputType) 2145 report_fatal_error("Input Type could not be determined."); 2146 2147 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2148 // Retreive the operation to use based on input type 2149 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2150 auto Opcode = 2151 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd : SPIRV::OpGroupNonUniformIAdd; 2152 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2153 .addDef(ResVReg) 2154 .addUse(GR.getSPIRVTypeID(ResType)) 2155 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2156 .addImm(SPIRV::GroupOperation::Reduce) 2157 .addUse(I.getOperand(2).getReg()); 2158 } 2159 2160 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2161 const SPIRVType *ResType, 2162 MachineInstr &I) const { 2163 MachineBasicBlock &BB = *I.getParent(); 2164 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2165 .addDef(ResVReg) 2166 .addUse(GR.getSPIRVTypeID(ResType)) 2167 .addUse(I.getOperand(1).getReg()) 2168 .constrainAllUses(TII, TRI, RBI); 2169 } 2170 2171 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2172 const SPIRVType *ResType, 2173 MachineInstr &I) const { 2174 // There is no way to implement `freeze` correctly without support on SPIR-V 2175 // standard side, but we may at least address a simple (static) case when 2176 // undef/poison value presence is obvious. The main benefit of even 2177 // incomplete `freeze` support is preventing of translation from crashing due 2178 // to lack of support on legalization and instruction selection steps. 2179 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2180 return false; 2181 Register OpReg = I.getOperand(1).getReg(); 2182 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2183 Register Reg; 2184 switch (Def->getOpcode()) { 2185 case SPIRV::ASSIGN_TYPE: 2186 if (MachineInstr *AssignToDef = 2187 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2188 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2189 Reg = Def->getOperand(2).getReg(); 2190 } 2191 break; 2192 case SPIRV::OpUndef: 2193 Reg = Def->getOperand(1).getReg(); 2194 break; 2195 } 2196 unsigned DestOpCode; 2197 if (Reg.isValid()) { 2198 DestOpCode = SPIRV::OpConstantNull; 2199 } else { 2200 DestOpCode = TargetOpcode::COPY; 2201 Reg = OpReg; 2202 } 2203 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2204 .addDef(I.getOperand(0).getReg()) 2205 .addUse(Reg) 2206 .constrainAllUses(TII, TRI, RBI); 2207 } 2208 return false; 2209 } 2210 2211 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2212 const SPIRVType *ResType) { 2213 Register OpReg = ResType->getOperand(2).getReg(); 2214 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2215 if (!OpDef) 2216 return 0; 2217 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2218 OpDef->getOperand(1).isReg()) { 2219 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2220 OpDef = RefDef; 2221 } 2222 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2223 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2224 : 0; 2225 return N; 2226 } 2227 2228 // Return true if the type represents a constant register 2229 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2230 SmallPtrSet<SPIRVType *, 4> &Visited) { 2231 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2232 OpDef->getOperand(1).isReg()) { 2233 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2234 OpDef = RefDef; 2235 } 2236 2237 if (Visited.contains(OpDef)) 2238 return true; 2239 Visited.insert(OpDef); 2240 2241 unsigned Opcode = OpDef->getOpcode(); 2242 switch (Opcode) { 2243 case TargetOpcode::G_CONSTANT: 2244 case TargetOpcode::G_FCONSTANT: 2245 return true; 2246 case TargetOpcode::G_INTRINSIC: 2247 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2248 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2249 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2250 Intrinsic::spv_const_composite; 2251 case TargetOpcode::G_BUILD_VECTOR: 2252 case TargetOpcode::G_SPLAT_VECTOR: { 2253 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2254 i++) { 2255 SPIRVType *OpNestedDef = 2256 OpDef->getOperand(i).isReg() 2257 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2258 : nullptr; 2259 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2260 return false; 2261 } 2262 return true; 2263 } 2264 } 2265 return false; 2266 } 2267 2268 // Return true if the virtual register represents a constant 2269 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2270 SmallPtrSet<SPIRVType *, 4> Visited; 2271 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2272 return isConstReg(MRI, OpDef, Visited); 2273 return false; 2274 } 2275 2276 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2277 const SPIRVType *ResType, 2278 MachineInstr &I) const { 2279 unsigned N = 0; 2280 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2281 N = GR.getScalarOrVectorComponentCount(ResType); 2282 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2283 N = getArrayComponentCount(MRI, ResType); 2284 else 2285 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2286 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2287 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2288 2289 // check if we may construct a constant vector 2290 bool IsConst = true; 2291 for (unsigned i = I.getNumExplicitDefs(); 2292 i < I.getNumExplicitOperands() && IsConst; ++i) 2293 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2294 IsConst = false; 2295 2296 if (!IsConst && N < 2) 2297 report_fatal_error( 2298 "There must be at least two constituent operands in a vector"); 2299 2300 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2301 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2302 TII.get(IsConst ? SPIRV::OpConstantComposite 2303 : SPIRV::OpCompositeConstruct)) 2304 .addDef(ResVReg) 2305 .addUse(GR.getSPIRVTypeID(ResType)); 2306 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2307 MIB.addUse(I.getOperand(i).getReg()); 2308 return MIB.constrainAllUses(TII, TRI, RBI); 2309 } 2310 2311 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2312 const SPIRVType *ResType, 2313 MachineInstr &I) const { 2314 unsigned N = 0; 2315 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2316 N = GR.getScalarOrVectorComponentCount(ResType); 2317 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2318 N = getArrayComponentCount(MRI, ResType); 2319 else 2320 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2321 2322 unsigned OpIdx = I.getNumExplicitDefs(); 2323 if (!I.getOperand(OpIdx).isReg()) 2324 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2325 2326 // check if we may construct a constant vector 2327 Register OpReg = I.getOperand(OpIdx).getReg(); 2328 bool IsConst = isConstReg(MRI, OpReg); 2329 2330 if (!IsConst && N < 2) 2331 report_fatal_error( 2332 "There must be at least two constituent operands in a vector"); 2333 2334 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2335 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2336 TII.get(IsConst ? SPIRV::OpConstantComposite 2337 : SPIRV::OpCompositeConstruct)) 2338 .addDef(ResVReg) 2339 .addUse(GR.getSPIRVTypeID(ResType)); 2340 for (unsigned i = 0; i < N; ++i) 2341 MIB.addUse(OpReg); 2342 return MIB.constrainAllUses(TII, TRI, RBI); 2343 } 2344 2345 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2346 const SPIRVType *ResType, 2347 MachineInstr &I) const { 2348 2349 unsigned Opcode; 2350 2351 if (STI.canUseExtension( 2352 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2353 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2354 Opcode = SPIRV::OpDemoteToHelperInvocation; 2355 } else { 2356 Opcode = SPIRV::OpKill; 2357 // OpKill must be the last operation of any basic block. 2358 if (MachineInstr *NextI = I.getNextNode()) { 2359 GR.invalidateMachineInstr(NextI); 2360 NextI->removeFromParent(); 2361 } 2362 } 2363 2364 MachineBasicBlock &BB = *I.getParent(); 2365 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2366 .constrainAllUses(TII, TRI, RBI); 2367 } 2368 2369 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2370 const SPIRVType *ResType, 2371 unsigned CmpOpc, 2372 MachineInstr &I) const { 2373 Register Cmp0 = I.getOperand(2).getReg(); 2374 Register Cmp1 = I.getOperand(3).getReg(); 2375 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2376 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2377 "CMP operands should have the same type"); 2378 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2379 .addDef(ResVReg) 2380 .addUse(GR.getSPIRVTypeID(ResType)) 2381 .addUse(Cmp0) 2382 .addUse(Cmp1) 2383 .constrainAllUses(TII, TRI, RBI); 2384 } 2385 2386 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2387 const SPIRVType *ResType, 2388 MachineInstr &I) const { 2389 auto Pred = I.getOperand(1).getPredicate(); 2390 unsigned CmpOpc; 2391 2392 Register CmpOperand = I.getOperand(2).getReg(); 2393 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2394 CmpOpc = getPtrCmpOpcode(Pred); 2395 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2396 CmpOpc = getBoolCmpOpcode(Pred); 2397 else 2398 CmpOpc = getICmpOpcode(Pred); 2399 return selectCmp(ResVReg, ResType, CmpOpc, I); 2400 } 2401 2402 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2403 const MachineInstr &I, 2404 int OpIdx) const { 2405 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2406 "Expected G_FCONSTANT"); 2407 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2408 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2409 } 2410 2411 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2412 const MachineInstr &I, 2413 int OpIdx) const { 2414 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2415 "Expected G_CONSTANT"); 2416 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2417 } 2418 2419 std::pair<Register, bool> 2420 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2421 const SPIRVType *ResType) const { 2422 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2423 const SPIRVType *SpvI32Ty = 2424 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2425 // Find a constant in DT or build a new one. 2426 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2427 Register NewReg = GR.find(ConstInt, GR.CurMF); 2428 bool Result = true; 2429 if (!NewReg.isValid()) { 2430 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2431 GR.add(ConstInt, GR.CurMF, NewReg); 2432 MachineInstr *MI; 2433 MachineBasicBlock &BB = *I.getParent(); 2434 if (Val == 0) { 2435 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2436 .addDef(NewReg) 2437 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2438 } else { 2439 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2440 .addDef(NewReg) 2441 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2442 .addImm(APInt(32, Val).getZExtValue()); 2443 } 2444 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2445 } 2446 return {NewReg, Result}; 2447 } 2448 2449 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2450 const SPIRVType *ResType, 2451 MachineInstr &I) const { 2452 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2453 return selectCmp(ResVReg, ResType, CmpOp, I); 2454 } 2455 2456 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2457 MachineInstr &I) const { 2458 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2459 bool ZeroAsNull = STI.isOpenCLEnv(); 2460 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2461 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2462 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2463 } 2464 2465 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2466 MachineInstr &I) const { 2467 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2468 bool ZeroAsNull = STI.isOpenCLEnv(); 2469 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2470 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2471 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2472 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2473 } 2474 2475 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2476 MachineInstr &I) const { 2477 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2478 bool ZeroAsNull = STI.isOpenCLEnv(); 2479 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2480 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2481 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2482 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2483 } 2484 2485 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2486 const SPIRVType *ResType, 2487 MachineInstr &I) const { 2488 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2489 APInt One = 2490 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2491 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2492 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2493 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2494 } 2495 2496 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2497 const SPIRVType *ResType, 2498 MachineInstr &I, 2499 bool IsSigned) const { 2500 // To extend a bool, we need to use OpSelect between constants. 2501 Register ZeroReg = buildZerosVal(ResType, I); 2502 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2503 bool IsScalarBool = 2504 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2505 unsigned Opcode = 2506 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2507 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2508 .addDef(ResVReg) 2509 .addUse(GR.getSPIRVTypeID(ResType)) 2510 .addUse(I.getOperand(1).getReg()) 2511 .addUse(OneReg) 2512 .addUse(ZeroReg) 2513 .constrainAllUses(TII, TRI, RBI); 2514 } 2515 2516 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2517 const SPIRVType *ResType, 2518 MachineInstr &I, bool IsSigned, 2519 unsigned Opcode) const { 2520 Register SrcReg = I.getOperand(1).getReg(); 2521 // We can convert bool value directly to float type without OpConvert*ToF, 2522 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2523 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2524 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2525 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2526 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2527 const unsigned NumElts = ResType->getOperand(2).getImm(); 2528 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2529 } 2530 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2531 selectSelect(SrcReg, TmpType, I, false); 2532 } 2533 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2534 } 2535 2536 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2537 const SPIRVType *ResType, 2538 MachineInstr &I, bool IsSigned) const { 2539 Register SrcReg = I.getOperand(1).getReg(); 2540 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2541 return selectSelect(ResVReg, ResType, I, IsSigned); 2542 2543 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2544 if (SrcType == ResType) 2545 return BuildCOPY(ResVReg, SrcReg, I); 2546 2547 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2548 return selectUnOp(ResVReg, ResType, I, Opcode); 2549 } 2550 2551 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2552 const SPIRVType *ResType, 2553 MachineInstr &I, 2554 bool IsSigned) const { 2555 MachineIRBuilder MIRBuilder(I); 2556 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2557 MachineBasicBlock &BB = *I.getParent(); 2558 // Ensure we have bool. 2559 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2560 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2561 if (N > 1) 2562 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2563 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2564 // Build less-than-equal and less-than. 2565 // TODO: replace with one-liner createVirtualRegister() from 2566 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2567 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2568 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2569 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2570 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2571 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2572 : SPIRV::OpULessThanEqual)) 2573 .addDef(IsLessEqReg) 2574 .addUse(BoolTypeReg) 2575 .addUse(I.getOperand(1).getReg()) 2576 .addUse(I.getOperand(2).getReg()) 2577 .constrainAllUses(TII, TRI, RBI); 2578 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2579 MRI->setType(IsLessReg, LLT::scalar(64)); 2580 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2581 Result &= BuildMI(BB, I, I.getDebugLoc(), 2582 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2583 .addDef(IsLessReg) 2584 .addUse(BoolTypeReg) 2585 .addUse(I.getOperand(1).getReg()) 2586 .addUse(I.getOperand(2).getReg()) 2587 .constrainAllUses(TII, TRI, RBI); 2588 // Build selects. 2589 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2590 Register NegOneOrZeroReg = 2591 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2592 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2593 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2594 unsigned SelectOpcode = 2595 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2596 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2597 .addDef(NegOneOrZeroReg) 2598 .addUse(ResTypeReg) 2599 .addUse(IsLessReg) 2600 .addUse(buildOnesVal(true, ResType, I)) // -1 2601 .addUse(buildZerosVal(ResType, I)) 2602 .constrainAllUses(TII, TRI, RBI); 2603 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2604 .addDef(ResVReg) 2605 .addUse(ResTypeReg) 2606 .addUse(IsLessEqReg) 2607 .addUse(NegOneOrZeroReg) // -1 or 0 2608 .addUse(buildOnesVal(false, ResType, I)) 2609 .constrainAllUses(TII, TRI, RBI); 2610 } 2611 2612 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2613 Register ResVReg, 2614 MachineInstr &I, 2615 const SPIRVType *IntTy, 2616 const SPIRVType *BoolTy) const { 2617 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2618 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2619 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2620 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2621 Register Zero = buildZerosVal(IntTy, I); 2622 Register One = buildOnesVal(false, IntTy, I); 2623 MachineBasicBlock &BB = *I.getParent(); 2624 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2625 .addDef(BitIntReg) 2626 .addUse(GR.getSPIRVTypeID(IntTy)) 2627 .addUse(IntReg) 2628 .addUse(One) 2629 .constrainAllUses(TII, TRI, RBI); 2630 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2631 .addDef(ResVReg) 2632 .addUse(GR.getSPIRVTypeID(BoolTy)) 2633 .addUse(BitIntReg) 2634 .addUse(Zero) 2635 .constrainAllUses(TII, TRI, RBI); 2636 } 2637 2638 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2639 const SPIRVType *ResType, 2640 MachineInstr &I) const { 2641 Register IntReg = I.getOperand(1).getReg(); 2642 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2643 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2644 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2645 if (ArgType == ResType) 2646 return BuildCOPY(ResVReg, IntReg, I); 2647 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2648 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2649 return selectUnOp(ResVReg, ResType, I, Opcode); 2650 } 2651 2652 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2653 const SPIRVType *ResType, 2654 const APInt &Imm, 2655 MachineInstr &I) const { 2656 unsigned TyOpcode = ResType->getOpcode(); 2657 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2658 MachineBasicBlock &BB = *I.getParent(); 2659 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2660 Imm.isZero()) 2661 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2662 .addDef(ResVReg) 2663 .addUse(GR.getSPIRVTypeID(ResType)) 2664 .constrainAllUses(TII, TRI, RBI); 2665 if (TyOpcode == SPIRV::OpTypeInt) { 2666 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2667 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2668 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2669 } 2670 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2671 .addDef(ResVReg) 2672 .addUse(GR.getSPIRVTypeID(ResType)); 2673 // <=32-bit integers should be caught by the sdag pattern. 2674 assert(Imm.getBitWidth() > 32); 2675 addNumImm(Imm, MIB); 2676 return MIB.constrainAllUses(TII, TRI, RBI); 2677 } 2678 2679 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2680 const SPIRVType *ResType, 2681 MachineInstr &I) const { 2682 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2683 .addDef(ResVReg) 2684 .addUse(GR.getSPIRVTypeID(ResType)) 2685 .constrainAllUses(TII, TRI, RBI); 2686 } 2687 2688 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2689 assert(MO.isReg()); 2690 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2691 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2692 assert(TypeInst->getOperand(1).isReg()); 2693 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2694 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2695 } 2696 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2697 } 2698 2699 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2700 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2701 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2702 return TypeInst->getOperand(2).getImm(); 2703 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2704 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2705 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2706 } 2707 2708 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2709 const SPIRVType *ResType, 2710 MachineInstr &I) const { 2711 MachineBasicBlock &BB = *I.getParent(); 2712 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2713 .addDef(ResVReg) 2714 .addUse(GR.getSPIRVTypeID(ResType)) 2715 // object to insert 2716 .addUse(I.getOperand(3).getReg()) 2717 // composite to insert into 2718 .addUse(I.getOperand(2).getReg()); 2719 for (unsigned i = 4; i < I.getNumOperands(); i++) 2720 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2721 return MIB.constrainAllUses(TII, TRI, RBI); 2722 } 2723 2724 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2725 const SPIRVType *ResType, 2726 MachineInstr &I) const { 2727 MachineBasicBlock &BB = *I.getParent(); 2728 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2729 .addDef(ResVReg) 2730 .addUse(GR.getSPIRVTypeID(ResType)) 2731 .addUse(I.getOperand(2).getReg()); 2732 for (unsigned i = 3; i < I.getNumOperands(); i++) 2733 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2734 return MIB.constrainAllUses(TII, TRI, RBI); 2735 } 2736 2737 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2738 const SPIRVType *ResType, 2739 MachineInstr &I) const { 2740 if (isImm(I.getOperand(4), MRI)) 2741 return selectInsertVal(ResVReg, ResType, I); 2742 MachineBasicBlock &BB = *I.getParent(); 2743 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2744 .addDef(ResVReg) 2745 .addUse(GR.getSPIRVTypeID(ResType)) 2746 .addUse(I.getOperand(2).getReg()) 2747 .addUse(I.getOperand(3).getReg()) 2748 .addUse(I.getOperand(4).getReg()) 2749 .constrainAllUses(TII, TRI, RBI); 2750 } 2751 2752 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2753 const SPIRVType *ResType, 2754 MachineInstr &I) const { 2755 if (isImm(I.getOperand(3), MRI)) 2756 return selectExtractVal(ResVReg, ResType, I); 2757 MachineBasicBlock &BB = *I.getParent(); 2758 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2759 .addDef(ResVReg) 2760 .addUse(GR.getSPIRVTypeID(ResType)) 2761 .addUse(I.getOperand(2).getReg()) 2762 .addUse(I.getOperand(3).getReg()) 2763 .constrainAllUses(TII, TRI, RBI); 2764 } 2765 2766 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2767 const SPIRVType *ResType, 2768 MachineInstr &I) const { 2769 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2770 2771 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2772 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2773 // we have to use Op[InBounds]AccessChain. 2774 const unsigned Opcode = STI.isVulkanEnv() 2775 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2776 : SPIRV::OpAccessChain) 2777 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2778 : SPIRV::OpPtrAccessChain); 2779 2780 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2781 .addDef(ResVReg) 2782 .addUse(GR.getSPIRVTypeID(ResType)) 2783 // Object to get a pointer to. 2784 .addUse(I.getOperand(3).getReg()); 2785 // Adding indices. 2786 const unsigned StartingIndex = 2787 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2788 ? 5 2789 : 4; 2790 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2791 Res.addUse(I.getOperand(i).getReg()); 2792 return Res.constrainAllUses(TII, TRI, RBI); 2793 } 2794 2795 // Maybe wrap a value into OpSpecConstantOp 2796 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2797 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2798 bool Result = true; 2799 unsigned Lim = I.getNumExplicitOperands(); 2800 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2801 Register OpReg = I.getOperand(i).getReg(); 2802 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2803 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2804 SmallPtrSet<SPIRVType *, 4> Visited; 2805 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2806 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2807 GR.isAggregateType(OpType)) { 2808 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2809 // by selectAddrSpaceCast() 2810 CompositeArgs.push_back(OpReg); 2811 continue; 2812 } 2813 MachineFunction *MF = I.getMF(); 2814 Register WrapReg = GR.find(OpDefine, MF); 2815 if (WrapReg.isValid()) { 2816 CompositeArgs.push_back(WrapReg); 2817 continue; 2818 } 2819 // Create a new register for the wrapper 2820 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2821 GR.add(OpDefine, MF, WrapReg); 2822 CompositeArgs.push_back(WrapReg); 2823 // Decorate the wrapper register and generate a new instruction 2824 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2825 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2826 MachineBasicBlock &BB = *I.getParent(); 2827 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2828 .addDef(WrapReg) 2829 .addUse(GR.getSPIRVTypeID(OpType)) 2830 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2831 .addUse(OpReg) 2832 .constrainAllUses(TII, TRI, RBI); 2833 if (!Result) 2834 break; 2835 } 2836 return Result; 2837 } 2838 2839 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2840 const SPIRVType *ResType, 2841 MachineInstr &I) const { 2842 MachineBasicBlock &BB = *I.getParent(); 2843 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2844 switch (IID) { 2845 case Intrinsic::spv_load: 2846 return selectLoad(ResVReg, ResType, I); 2847 case Intrinsic::spv_store: 2848 return selectStore(I); 2849 case Intrinsic::spv_extractv: 2850 return selectExtractVal(ResVReg, ResType, I); 2851 case Intrinsic::spv_insertv: 2852 return selectInsertVal(ResVReg, ResType, I); 2853 case Intrinsic::spv_extractelt: 2854 return selectExtractElt(ResVReg, ResType, I); 2855 case Intrinsic::spv_insertelt: 2856 return selectInsertElt(ResVReg, ResType, I); 2857 case Intrinsic::spv_gep: 2858 return selectGEP(ResVReg, ResType, I); 2859 case Intrinsic::spv_unref_global: 2860 case Intrinsic::spv_init_global: { 2861 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2862 MachineInstr *Init = I.getNumExplicitOperands() > 2 2863 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2864 : nullptr; 2865 assert(MI); 2866 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2867 } 2868 case Intrinsic::spv_undef: { 2869 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2870 .addDef(ResVReg) 2871 .addUse(GR.getSPIRVTypeID(ResType)); 2872 return MIB.constrainAllUses(TII, TRI, RBI); 2873 } 2874 case Intrinsic::spv_const_composite: { 2875 // If no values are attached, the composite is null constant. 2876 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2877 // Select a proper instruction. 2878 unsigned Opcode = SPIRV::OpConstantNull; 2879 SmallVector<Register> CompositeArgs; 2880 if (!IsNull) { 2881 Opcode = SPIRV::OpConstantComposite; 2882 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2883 return false; 2884 } 2885 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2886 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2887 .addDef(ResVReg) 2888 .addUse(GR.getSPIRVTypeID(ResType)); 2889 // skip type MD node we already used when generated assign.type for this 2890 if (!IsNull) { 2891 for (Register OpReg : CompositeArgs) 2892 MIB.addUse(OpReg); 2893 } 2894 return MIB.constrainAllUses(TII, TRI, RBI); 2895 } 2896 case Intrinsic::spv_assign_name: { 2897 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2898 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2899 for (unsigned i = I.getNumExplicitDefs() + 2; 2900 i < I.getNumExplicitOperands(); ++i) { 2901 MIB.addImm(I.getOperand(i).getImm()); 2902 } 2903 return MIB.constrainAllUses(TII, TRI, RBI); 2904 } 2905 case Intrinsic::spv_switch: { 2906 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2907 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2908 if (I.getOperand(i).isReg()) 2909 MIB.addReg(I.getOperand(i).getReg()); 2910 else if (I.getOperand(i).isCImm()) 2911 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2912 else if (I.getOperand(i).isMBB()) 2913 MIB.addMBB(I.getOperand(i).getMBB()); 2914 else 2915 llvm_unreachable("Unexpected OpSwitch operand"); 2916 } 2917 return MIB.constrainAllUses(TII, TRI, RBI); 2918 } 2919 case Intrinsic::spv_loop_merge: { 2920 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge)); 2921 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2922 assert(I.getOperand(i).isMBB()); 2923 MIB.addMBB(I.getOperand(i).getMBB()); 2924 } 2925 MIB.addImm(SPIRV::SelectionControl::None); 2926 return MIB.constrainAllUses(TII, TRI, RBI); 2927 } 2928 case Intrinsic::spv_selection_merge: { 2929 auto MIB = 2930 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge)); 2931 assert(I.getOperand(1).isMBB() && 2932 "operand 1 to spv_selection_merge must be a basic block"); 2933 MIB.addMBB(I.getOperand(1).getMBB()); 2934 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm())); 2935 return MIB.constrainAllUses(TII, TRI, RBI); 2936 } 2937 case Intrinsic::spv_cmpxchg: 2938 return selectAtomicCmpXchg(ResVReg, ResType, I); 2939 case Intrinsic::spv_unreachable: 2940 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2941 .constrainAllUses(TII, TRI, RBI); 2942 case Intrinsic::spv_alloca: 2943 return selectFrameIndex(ResVReg, ResType, I); 2944 case Intrinsic::spv_alloca_array: 2945 return selectAllocaArray(ResVReg, ResType, I); 2946 case Intrinsic::spv_assume: 2947 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2948 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2949 .addUse(I.getOperand(1).getReg()) 2950 .constrainAllUses(TII, TRI, RBI); 2951 break; 2952 case Intrinsic::spv_expect: 2953 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2954 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2955 .addDef(ResVReg) 2956 .addUse(GR.getSPIRVTypeID(ResType)) 2957 .addUse(I.getOperand(2).getReg()) 2958 .addUse(I.getOperand(3).getReg()) 2959 .constrainAllUses(TII, TRI, RBI); 2960 break; 2961 case Intrinsic::arithmetic_fence: 2962 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2963 return BuildMI(BB, I, I.getDebugLoc(), 2964 TII.get(SPIRV::OpArithmeticFenceEXT)) 2965 .addDef(ResVReg) 2966 .addUse(GR.getSPIRVTypeID(ResType)) 2967 .addUse(I.getOperand(2).getReg()) 2968 .constrainAllUses(TII, TRI, RBI); 2969 else 2970 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2971 break; 2972 case Intrinsic::spv_thread_id: 2973 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 2974 // intrinsic in LLVM IR for SPIR-V backend. 2975 // 2976 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 2977 // `GlobalInvocationId` builtin variable 2978 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 2979 ResType, I); 2980 case Intrinsic::spv_thread_id_in_group: 2981 // The HLSL SV_GroupThreadId semantic is lowered to 2982 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 2983 // 2984 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 2985 // translated to a `LocalInvocationId` builtin variable 2986 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 2987 ResType, I); 2988 case Intrinsic::spv_group_id: 2989 // The HLSL SV_GroupId semantic is lowered to 2990 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. 2991 // 2992 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` 2993 // builtin variable 2994 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, 2995 I); 2996 case Intrinsic::spv_fdot: 2997 return selectFloatDot(ResVReg, ResType, I); 2998 case Intrinsic::spv_udot: 2999 case Intrinsic::spv_sdot: 3000 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3001 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3002 return selectIntegerDot(ResVReg, ResType, I, 3003 /*Signed=*/IID == Intrinsic::spv_sdot); 3004 return selectIntegerDotExpansion(ResVReg, ResType, I); 3005 case Intrinsic::spv_dot4add_i8packed: 3006 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3007 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3008 return selectDot4AddPacked<true>(ResVReg, ResType, I); 3009 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 3010 case Intrinsic::spv_dot4add_u8packed: 3011 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 3012 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 3013 return selectDot4AddPacked<false>(ResVReg, ResType, I); 3014 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 3015 case Intrinsic::spv_all: 3016 return selectAll(ResVReg, ResType, I); 3017 case Intrinsic::spv_any: 3018 return selectAny(ResVReg, ResType, I); 3019 case Intrinsic::spv_cross: 3020 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 3021 case Intrinsic::spv_distance: 3022 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); 3023 case Intrinsic::spv_lerp: 3024 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 3025 case Intrinsic::spv_length: 3026 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 3027 case Intrinsic::spv_degrees: 3028 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 3029 case Intrinsic::spv_frac: 3030 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 3031 case Intrinsic::spv_normalize: 3032 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 3033 case Intrinsic::spv_rsqrt: 3034 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 3035 case Intrinsic::spv_sign: 3036 return selectSign(ResVReg, ResType, I); 3037 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 3038 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 3039 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 3040 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 3041 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb 3042 return selectFirstBitLow(ResVReg, ResType, I); 3043 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 3044 bool Result = true; 3045 auto MemSemConstant = 3046 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 3047 Register MemSemReg = MemSemConstant.first; 3048 Result &= MemSemConstant.second; 3049 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 3050 Register ScopeReg = ScopeConstant.first; 3051 Result &= ScopeConstant.second; 3052 MachineBasicBlock &BB = *I.getParent(); 3053 return Result && 3054 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 3055 .addUse(ScopeReg) 3056 .addUse(ScopeReg) 3057 .addUse(MemSemReg) 3058 .constrainAllUses(TII, TRI, RBI); 3059 } 3060 case Intrinsic::spv_lifetime_start: 3061 case Intrinsic::spv_lifetime_end: { 3062 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 3063 : SPIRV::OpLifetimeStop; 3064 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 3065 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 3066 if (Size == -1) 3067 Size = 0; 3068 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 3069 .addUse(PtrReg) 3070 .addImm(Size) 3071 .constrainAllUses(TII, TRI, RBI); 3072 } 3073 case Intrinsic::spv_saturate: 3074 return selectSaturate(ResVReg, ResType, I); 3075 case Intrinsic::spv_nclamp: 3076 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 3077 case Intrinsic::spv_uclamp: 3078 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 3079 case Intrinsic::spv_sclamp: 3080 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 3081 case Intrinsic::spv_wave_active_countbits: 3082 return selectWaveActiveCountBits(ResVReg, ResType, I); 3083 case Intrinsic::spv_wave_all: 3084 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 3085 case Intrinsic::spv_wave_any: 3086 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 3087 case Intrinsic::spv_wave_is_first_lane: 3088 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 3089 case Intrinsic::spv_wave_reduce_sum: 3090 return selectWaveReduceSum(ResVReg, ResType, I); 3091 case Intrinsic::spv_wave_readlane: 3092 return selectWaveOpInst(ResVReg, ResType, I, 3093 SPIRV::OpGroupNonUniformShuffle); 3094 case Intrinsic::spv_step: 3095 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 3096 case Intrinsic::spv_radians: 3097 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 3098 // Discard intrinsics which we do not expect to actually represent code after 3099 // lowering or intrinsics which are not implemented but should not crash when 3100 // found in a customer's LLVM IR input. 3101 case Intrinsic::instrprof_increment: 3102 case Intrinsic::instrprof_increment_step: 3103 case Intrinsic::instrprof_value_profile: 3104 break; 3105 // Discard internal intrinsics. 3106 case Intrinsic::spv_value_md: 3107 break; 3108 case Intrinsic::spv_resource_handlefrombinding: { 3109 return selectHandleFromBinding(ResVReg, ResType, I); 3110 } 3111 case Intrinsic::spv_resource_store_typedbuffer: { 3112 return selectImageWriteIntrinsic(I); 3113 } 3114 case Intrinsic::spv_resource_load_typedbuffer: { 3115 return selectReadImageIntrinsic(ResVReg, ResType, I); 3116 } 3117 case Intrinsic::spv_resource_getpointer: { 3118 return selectResourceGetPointer(ResVReg, ResType, I); 3119 } 3120 case Intrinsic::spv_discard: { 3121 return selectDiscard(ResVReg, ResType, I); 3122 } 3123 default: { 3124 std::string DiagMsg; 3125 raw_string_ostream OS(DiagMsg); 3126 I.print(OS); 3127 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3128 report_fatal_error(DiagMsg.c_str(), false); 3129 } 3130 } 3131 return true; 3132 } 3133 3134 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3135 const SPIRVType *ResType, 3136 MachineInstr &I) const { 3137 return true; 3138 } 3139 3140 bool SPIRVInstructionSelector::selectReadImageIntrinsic( 3141 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3142 3143 // If the load of the image is in a different basic block, then 3144 // this will generate invalid code. A proper solution is to move 3145 // the OpLoad from selectHandleFromBinding here. However, to do 3146 // that we will need to change the return type of the intrinsic. 3147 // We will do that when we can, but for now trying to move forward with other 3148 // issues. 3149 Register ImageReg = I.getOperand(2).getReg(); 3150 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 3151 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 3152 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 3153 *ImageDef, I)) { 3154 return false; 3155 } 3156 3157 Register IdxReg = I.getOperand(3).getReg(); 3158 DebugLoc Loc = I.getDebugLoc(); 3159 MachineInstr &Pos = I; 3160 3161 return generateImageRead(ResVReg, ResType, NewImageReg, IdxReg, Loc, Pos); 3162 } 3163 3164 bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg, 3165 const SPIRVType *ResType, 3166 Register ImageReg, 3167 Register IdxReg, DebugLoc Loc, 3168 MachineInstr &Pos) const { 3169 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3170 if (ResultSize == 4) { 3171 return BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead)) 3172 .addDef(ResVReg) 3173 .addUse(GR.getSPIRVTypeID(ResType)) 3174 .addUse(ImageReg) 3175 .addUse(IdxReg) 3176 .constrainAllUses(TII, TRI, RBI); 3177 } 3178 3179 SPIRVType *ReadType = widenTypeToVec4(ResType, Pos); 3180 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3181 bool Succeed = 3182 BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead)) 3183 .addDef(ReadReg) 3184 .addUse(GR.getSPIRVTypeID(ReadType)) 3185 .addUse(ImageReg) 3186 .addUse(IdxReg) 3187 .constrainAllUses(TII, TRI, RBI); 3188 if (!Succeed) 3189 return false; 3190 3191 if (ResultSize == 1) { 3192 return BuildMI(*Pos.getParent(), Pos, Loc, 3193 TII.get(SPIRV::OpCompositeExtract)) 3194 .addDef(ResVReg) 3195 .addUse(GR.getSPIRVTypeID(ResType)) 3196 .addUse(ReadReg) 3197 .addImm(0) 3198 .constrainAllUses(TII, TRI, RBI); 3199 } 3200 return extractSubvector(ResVReg, ResType, ReadReg, Pos); 3201 } 3202 3203 bool SPIRVInstructionSelector::selectResourceGetPointer( 3204 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3205 #ifdef ASSERT 3206 // For now, the operand is an image. This will change once we start handling 3207 // more resource types. 3208 Register ResourcePtr = I.getOperand(2).getReg(); 3209 SPIRVType *RegType = GR.getResultType(ResourcePtr); 3210 assert(RegType->getOpcode() == SPIRV::OpTypeImage && 3211 "Can only handle texel buffers for now."); 3212 #endif 3213 3214 // For texel buffers, the index into the image is part of the OpImageRead or 3215 // OpImageWrite instructions. So we will do nothing in this case. This 3216 // intrinsic will be combined with the load or store when selecting the load 3217 // or store. 3218 return true; 3219 } 3220 3221 bool SPIRVInstructionSelector::extractSubvector( 3222 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3223 MachineInstr &InsertionPoint) const { 3224 SPIRVType *InputType = GR.getResultType(ReadReg); 3225 [[maybe_unused]] uint64_t InputSize = 3226 GR.getScalarOrVectorComponentCount(InputType); 3227 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3228 assert(InputSize > 1 && "The input must be a vector."); 3229 assert(ResultSize > 1 && "The result must be a vector."); 3230 assert(ResultSize < InputSize && 3231 "Cannot extract more element than there are in the input."); 3232 SmallVector<Register> ComponentRegisters; 3233 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3234 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3235 for (uint64_t I = 0; I < ResultSize; I++) { 3236 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3237 bool Succeed = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3238 InsertionPoint.getDebugLoc(), 3239 TII.get(SPIRV::OpCompositeExtract)) 3240 .addDef(ComponentReg) 3241 .addUse(ScalarType->getOperand(0).getReg()) 3242 .addUse(ReadReg) 3243 .addImm(I) 3244 .constrainAllUses(TII, TRI, RBI); 3245 if (!Succeed) 3246 return false; 3247 ComponentRegisters.emplace_back(ComponentReg); 3248 } 3249 3250 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3251 InsertionPoint.getDebugLoc(), 3252 TII.get(SPIRV::OpCompositeConstruct)) 3253 .addDef(ResVReg) 3254 .addUse(GR.getSPIRVTypeID(ResType)); 3255 3256 for (Register ComponentReg : ComponentRegisters) 3257 MIB.addUse(ComponentReg); 3258 return MIB.constrainAllUses(TII, TRI, RBI); 3259 } 3260 3261 bool SPIRVInstructionSelector::selectImageWriteIntrinsic( 3262 MachineInstr &I) const { 3263 // If the load of the image is in a different basic block, then 3264 // this will generate invalid code. A proper solution is to move 3265 // the OpLoad from selectHandleFromBinding here. However, to do 3266 // that we will need to change the return type of the intrinsic. 3267 // We will do that when we can, but for now trying to move forward with other 3268 // issues. 3269 Register ImageReg = I.getOperand(1).getReg(); 3270 auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); 3271 Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); 3272 if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), 3273 *ImageDef, I)) { 3274 return false; 3275 } 3276 3277 Register CoordinateReg = I.getOperand(2).getReg(); 3278 Register DataReg = I.getOperand(3).getReg(); 3279 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3280 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3281 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3282 TII.get(SPIRV::OpImageWrite)) 3283 .addUse(NewImageReg) 3284 .addUse(CoordinateReg) 3285 .addUse(DataReg) 3286 .constrainAllUses(TII, TRI, RBI); 3287 } 3288 3289 Register SPIRVInstructionSelector::buildPointerToResource( 3290 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3291 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3292 MachineIRBuilder MIRBuilder) const { 3293 if (ArraySize == 1) 3294 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3295 MIRBuilder); 3296 3297 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3298 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3299 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3300 VarType, Set, Binding, MIRBuilder); 3301 3302 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3303 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3304 3305 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3306 if (IsNonUniform) { 3307 // It is unclear which value needs to be marked an non-uniform, so both 3308 // the index and the access changed are decorated as non-uniform. 3309 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3310 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3311 } 3312 3313 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3314 .addDef(AcReg) 3315 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3316 .addUse(VarReg) 3317 .addUse(IndexReg); 3318 3319 return AcReg; 3320 } 3321 3322 bool SPIRVInstructionSelector::selectFirstBitSet16( 3323 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3324 unsigned ExtendOpcode, unsigned BitSetOpcode) const { 3325 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3326 bool Result = selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, 3327 ExtendOpcode); 3328 3329 return Result && 3330 selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode); 3331 } 3332 3333 bool SPIRVInstructionSelector::selectFirstBitSet32( 3334 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3335 Register SrcReg, unsigned BitSetOpcode) const { 3336 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3337 .addDef(ResVReg) 3338 .addUse(GR.getSPIRVTypeID(ResType)) 3339 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3340 .addImm(BitSetOpcode) 3341 .addUse(SrcReg) 3342 .constrainAllUses(TII, TRI, RBI); 3343 } 3344 3345 bool SPIRVInstructionSelector::selectFirstBitSet64Overflow( 3346 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3347 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3348 3349 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors 3350 // requires creating a param register and return register with an invalid 3351 // vector size. If that is resolved, then this function can be used for 3352 // vectors of any component size. 3353 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3354 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops"); 3355 3356 MachineIRBuilder MIRBuilder(I); 3357 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3358 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder); 3359 SPIRVType *I64x2Type = GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder); 3360 SPIRVType *Vec2ResType = 3361 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder); 3362 3363 std::vector<Register> PartialRegs; 3364 3365 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd 3366 unsigned CurrentComponent = 0; 3367 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) { 3368 // This register holds the firstbitX result for each of the i64x2 vectors 3369 // extracted from SrcReg 3370 Register BitSetResult = 3371 MRI->createVirtualRegister(GR.getRegClass(I64x2Type)); 3372 3373 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3374 TII.get(SPIRV::OpVectorShuffle)) 3375 .addDef(BitSetResult) 3376 .addUse(GR.getSPIRVTypeID(I64x2Type)) 3377 .addUse(SrcReg) 3378 .addUse(SrcReg) 3379 .addImm(CurrentComponent) 3380 .addImm(CurrentComponent + 1); 3381 3382 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3383 return false; 3384 3385 Register SubVecBitSetReg = 3386 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType)); 3387 3388 if (!selectFirstBitSet64(SubVecBitSetReg, Vec2ResType, I, BitSetResult, 3389 BitSetOpcode, SwapPrimarySide)) 3390 return false; 3391 3392 PartialRegs.push_back(SubVecBitSetReg); 3393 } 3394 3395 // On odd component counts we need to handle one more component 3396 if (CurrentComponent != ComponentCount) { 3397 bool ZeroAsNull = STI.isOpenCLEnv(); 3398 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type)); 3399 Register ConstIntLastIdx = GR.getOrCreateConstInt( 3400 ComponentCount - 1, I, BaseType, TII, ZeroAsNull); 3401 3402 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx}, 3403 SPIRV::OpVectorExtractDynamic)) 3404 return false; 3405 3406 Register FinalElemBitSetReg = 3407 MRI->createVirtualRegister(GR.getRegClass(BaseType)); 3408 3409 if (!selectFirstBitSet64(FinalElemBitSetReg, BaseType, I, FinalElemReg, 3410 BitSetOpcode, SwapPrimarySide)) 3411 return false; 3412 3413 PartialRegs.push_back(FinalElemBitSetReg); 3414 } 3415 3416 // Join all the resulting registers back into the return type in order 3417 // (ie i32x2, i32x2, i32x1 -> i32x5) 3418 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs, 3419 SPIRV::OpCompositeConstruct); 3420 } 3421 3422 bool SPIRVInstructionSelector::selectFirstBitSet64( 3423 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3424 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3425 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3426 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3427 bool ZeroAsNull = STI.isOpenCLEnv(); 3428 Register ConstIntZero = 3429 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull); 3430 Register ConstIntOne = 3431 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull); 3432 3433 // SPIRV doesn't support vectors with more than 4 components. Since the 3434 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only 3435 // operate on vectors with 2 or less components. When largers vectors are 3436 // seen. Split them, recurse, then recombine them. 3437 if (ComponentCount > 2) { 3438 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg, 3439 BitSetOpcode, SwapPrimarySide); 3440 } 3441 3442 // 1. Split int64 into 2 pieces using a bitcast 3443 MachineIRBuilder MIRBuilder(I); 3444 SPIRVType *PostCastType = 3445 GR.getOrCreateSPIRVVectorType(BaseType, 2 * ComponentCount, MIRBuilder); 3446 Register BitcastReg = 3447 MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3448 3449 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg}, 3450 SPIRV::OpBitcast)) 3451 return false; 3452 3453 // 2. Find the first set bit from the primary side for all the pieces in #1 3454 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3455 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode)) 3456 return false; 3457 3458 // 3. Split result vector into high bits and low bits 3459 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3460 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3461 3462 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3463 if (IsScalarRes) { 3464 // if scalar do a vector extract 3465 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntZero}, 3466 SPIRV::OpVectorExtractDynamic)) 3467 return false; 3468 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntOne}, 3469 SPIRV::OpVectorExtractDynamic)) 3470 return false; 3471 } else { 3472 // if vector do a shufflevector 3473 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3474 TII.get(SPIRV::OpVectorShuffle)) 3475 .addDef(HighReg) 3476 .addUse(GR.getSPIRVTypeID(ResType)) 3477 .addUse(FBSReg) 3478 // Per the spec, repeat the vector if only one vec is needed 3479 .addUse(FBSReg); 3480 3481 // high bits are stored in even indexes. Extract them from FBSReg 3482 for (unsigned J = 0; J < ComponentCount * 2; J += 2) { 3483 MIB.addImm(J); 3484 } 3485 3486 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3487 return false; 3488 3489 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3490 TII.get(SPIRV::OpVectorShuffle)) 3491 .addDef(LowReg) 3492 .addUse(GR.getSPIRVTypeID(ResType)) 3493 .addUse(FBSReg) 3494 // Per the spec, repeat the vector if only one vec is needed 3495 .addUse(FBSReg); 3496 3497 // low bits are stored in odd indexes. Extract them from FBSReg 3498 for (unsigned J = 1; J < ComponentCount * 2; J += 2) { 3499 MIB.addImm(J); 3500 } 3501 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3502 return false; 3503 } 3504 3505 // 4. Check the result. When primary bits == -1 use secondary, otherwise use 3506 // primary 3507 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3508 Register NegOneReg; 3509 Register Reg0; 3510 Register Reg32; 3511 unsigned SelectOp; 3512 unsigned AddOp; 3513 3514 if (IsScalarRes) { 3515 NegOneReg = 3516 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3517 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3518 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3519 SelectOp = SPIRV::OpSelectSISCond; 3520 AddOp = SPIRV::OpIAddS; 3521 } else { 3522 BoolType = 3523 GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount, MIRBuilder); 3524 NegOneReg = 3525 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3526 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3527 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3528 SelectOp = SPIRV::OpSelectVIVCond; 3529 AddOp = SPIRV::OpIAddV; 3530 } 3531 3532 Register PrimaryReg = HighReg; 3533 Register SecondaryReg = LowReg; 3534 Register PrimaryShiftReg = Reg32; 3535 Register SecondaryShiftReg = Reg0; 3536 3537 // By default the emitted opcodes check for the set bit from the MSB side. 3538 // Setting SwapPrimarySide checks the set bit from the LSB side 3539 if (SwapPrimarySide) { 3540 PrimaryReg = LowReg; 3541 SecondaryReg = HighReg; 3542 PrimaryShiftReg = Reg0; 3543 SecondaryShiftReg = Reg32; 3544 } 3545 3546 // Check if the primary bits are == -1 3547 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3548 if (!selectOpWithSrcs(BReg, BoolType, I, {PrimaryReg, NegOneReg}, 3549 SPIRV::OpIEqual)) 3550 return false; 3551 3552 // Select secondary bits if true in BReg, otherwise primary bits 3553 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3554 if (!selectOpWithSrcs(TmpReg, ResType, I, {BReg, SecondaryReg, PrimaryReg}, 3555 SelectOp)) 3556 return false; 3557 3558 // 5. Add 32 when high bits are used, otherwise 0 for low bits 3559 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3560 if (!selectOpWithSrcs(ValReg, ResType, I, 3561 {BReg, SecondaryShiftReg, PrimaryShiftReg}, SelectOp)) 3562 return false; 3563 3564 return selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, AddOp); 3565 } 3566 3567 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3568 const SPIRVType *ResType, 3569 MachineInstr &I, 3570 bool IsSigned) const { 3571 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3572 Register OpReg = I.getOperand(2).getReg(); 3573 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3574 // zero or sign extend 3575 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3576 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3577 3578 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3579 case 16: 3580 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3581 case 32: 3582 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3583 case 64: 3584 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3585 /*SwapPrimarySide=*/false); 3586 default: 3587 report_fatal_error( 3588 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3589 } 3590 } 3591 3592 bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg, 3593 const SPIRVType *ResType, 3594 MachineInstr &I) const { 3595 // FindILsb intrinsic only supports 32 bit integers 3596 Register OpReg = I.getOperand(2).getReg(); 3597 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3598 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it 3599 // to an unsigned i32. As this leaves all the least significant bits unchanged 3600 // so the first set bit from the LSB side doesn't change. 3601 unsigned ExtendOpcode = SPIRV::OpUConvert; 3602 unsigned BitSetOpcode = GL::FindILsb; 3603 3604 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3605 case 16: 3606 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3607 case 32: 3608 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3609 case 64: 3610 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3611 /*SwapPrimarySide=*/true); 3612 default: 3613 report_fatal_error("spv_firstbitlow only supports 16,32,64 bits."); 3614 } 3615 } 3616 3617 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3618 const SPIRVType *ResType, 3619 MachineInstr &I) const { 3620 // there was an allocation size parameter to the allocation instruction 3621 // that is not 1 3622 MachineBasicBlock &BB = *I.getParent(); 3623 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3624 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3625 .addDef(ResVReg) 3626 .addUse(GR.getSPIRVTypeID(ResType)) 3627 .addUse(I.getOperand(2).getReg()) 3628 .constrainAllUses(TII, TRI, RBI); 3629 if (!STI.isVulkanEnv()) { 3630 unsigned Alignment = I.getOperand(3).getImm(); 3631 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3632 } 3633 return Res; 3634 } 3635 3636 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3637 const SPIRVType *ResType, 3638 MachineInstr &I) const { 3639 // Change order of instructions if needed: all OpVariable instructions in a 3640 // function must be the first instructions in the first block 3641 auto It = getOpVariableMBBIt(I); 3642 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3643 TII.get(SPIRV::OpVariable)) 3644 .addDef(ResVReg) 3645 .addUse(GR.getSPIRVTypeID(ResType)) 3646 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3647 .constrainAllUses(TII, TRI, RBI); 3648 if (!STI.isVulkanEnv()) { 3649 unsigned Alignment = I.getOperand(2).getImm(); 3650 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3651 {Alignment}); 3652 } 3653 return Res; 3654 } 3655 3656 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3657 // InstructionSelector walks backwards through the instructions. We can use 3658 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3659 // first, so can generate an OpBranchConditional here. If there is no 3660 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3661 const MachineInstr *PrevI = I.getPrevNode(); 3662 MachineBasicBlock &MBB = *I.getParent(); 3663 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3664 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3665 .addUse(PrevI->getOperand(0).getReg()) 3666 .addMBB(PrevI->getOperand(1).getMBB()) 3667 .addMBB(I.getOperand(0).getMBB()) 3668 .constrainAllUses(TII, TRI, RBI); 3669 } 3670 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3671 .addMBB(I.getOperand(0).getMBB()) 3672 .constrainAllUses(TII, TRI, RBI); 3673 } 3674 3675 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3676 // InstructionSelector walks backwards through the instructions. For an 3677 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3678 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3679 // generate the OpBranchConditional in selectBranch above. 3680 // 3681 // If an OpBranchConditional has been generated, we simply return, as the work 3682 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3683 // implicit fallthrough to the next basic block, so we need to create an 3684 // OpBranchConditional with an explicit "false" argument pointing to the next 3685 // basic block that LLVM would fall through to. 3686 const MachineInstr *NextI = I.getNextNode(); 3687 // Check if this has already been successfully selected. 3688 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3689 return true; 3690 // Must be relying on implicit block fallthrough, so generate an 3691 // OpBranchConditional with the "next" basic block as the "false" target. 3692 MachineBasicBlock &MBB = *I.getParent(); 3693 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3694 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3695 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3696 .addUse(I.getOperand(0).getReg()) 3697 .addMBB(I.getOperand(1).getMBB()) 3698 .addMBB(NextMBB) 3699 .constrainAllUses(TII, TRI, RBI); 3700 } 3701 3702 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3703 const SPIRVType *ResType, 3704 MachineInstr &I) const { 3705 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3706 .addDef(ResVReg) 3707 .addUse(GR.getSPIRVTypeID(ResType)); 3708 const unsigned NumOps = I.getNumOperands(); 3709 for (unsigned i = 1; i < NumOps; i += 2) { 3710 MIB.addUse(I.getOperand(i + 0).getReg()); 3711 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3712 } 3713 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3714 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3715 MIB->removeOperand(1); 3716 return Res; 3717 } 3718 3719 bool SPIRVInstructionSelector::selectGlobalValue( 3720 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3721 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3722 MachineIRBuilder MIRBuilder(I); 3723 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3724 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3725 SPIRVType *PointerBaseType; 3726 if (GVType->isArrayTy()) { 3727 SPIRVType *ArrayElementType = 3728 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3729 SPIRV::AccessQualifier::ReadWrite, false); 3730 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3731 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3732 } else { 3733 PointerBaseType = GR.getOrCreateSPIRVType( 3734 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3735 } 3736 3737 std::string GlobalIdent; 3738 if (!GV->hasName()) { 3739 unsigned &ID = UnnamedGlobalIDs[GV]; 3740 if (ID == 0) 3741 ID = UnnamedGlobalIDs.size(); 3742 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3743 } else { 3744 GlobalIdent = GV->getName(); 3745 } 3746 3747 // Behaviour of functions as operands depends on availability of the 3748 // corresponding extension (SPV_INTEL_function_pointers): 3749 // - If there is an extension to operate with functions as operands: 3750 // We create a proper constant operand and evaluate a correct type for a 3751 // function pointer. 3752 // - Without the required extension: 3753 // We have functions as operands in tests with blocks of instruction e.g. in 3754 // transcoding/global_block.ll. These operands are not used and should be 3755 // substituted by zero constants. Their type is expected to be always 3756 // OpTypePointer Function %uchar. 3757 if (isa<Function>(GV)) { 3758 const Constant *ConstVal = GV; 3759 MachineBasicBlock &BB = *I.getParent(); 3760 Register NewReg = GR.find(ConstVal, GR.CurMF); 3761 if (!NewReg.isValid()) { 3762 Register NewReg = ResVReg; 3763 GR.add(ConstVal, GR.CurMF, NewReg); 3764 const Function *GVFun = 3765 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3766 ? dyn_cast<Function>(GV) 3767 : nullptr; 3768 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3769 PointerBaseType, I, TII, 3770 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3771 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3772 if (GVFun) { 3773 // References to a function via function pointers generate virtual 3774 // registers without a definition. We will resolve it later, during 3775 // module analysis stage. 3776 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 3777 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3778 Register FuncVReg = 3779 MRI->createGenericVirtualRegister(GR.getRegType(ResType)); 3780 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); 3781 MachineInstrBuilder MIB1 = 3782 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 3783 .addDef(FuncVReg) 3784 .addUse(ResTypeReg); 3785 MachineInstrBuilder MIB2 = 3786 BuildMI(BB, I, I.getDebugLoc(), 3787 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3788 .addDef(NewReg) 3789 .addUse(ResTypeReg) 3790 .addUse(FuncVReg); 3791 // mapping the function pointer to the used Function 3792 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); 3793 return MIB1.constrainAllUses(TII, TRI, RBI) && 3794 MIB2.constrainAllUses(TII, TRI, RBI); 3795 } 3796 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3797 .addDef(NewReg) 3798 .addUse(GR.getSPIRVTypeID(ResType)) 3799 .constrainAllUses(TII, TRI, RBI); 3800 } 3801 assert(NewReg != ResVReg); 3802 return BuildCOPY(ResVReg, NewReg, I); 3803 } 3804 auto GlobalVar = cast<GlobalVariable>(GV); 3805 assert(GlobalVar->getName() != "llvm.global.annotations"); 3806 3807 // Skip empty declaration for GVs with initializers till we get the decl with 3808 // passed initializer. 3809 if (hasInitializer(GlobalVar) && !Init) 3810 return true; 3811 3812 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage(); 3813 SPIRV::LinkageType::LinkageType LnkType = 3814 GV->isDeclarationForLinker() 3815 ? SPIRV::LinkageType::Import 3816 : (GV->hasLinkOnceODRLinkage() && 3817 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3818 ? SPIRV::LinkageType::LinkOnceODR 3819 : SPIRV::LinkageType::Export); 3820 3821 const unsigned AddrSpace = GV->getAddressSpace(); 3822 SPIRV::StorageClass::StorageClass StorageClass = 3823 addressSpaceToStorageClass(AddrSpace, STI); 3824 SPIRVType *ResType = 3825 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3826 Register Reg = GR.buildGlobalVariable( 3827 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3828 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3829 return Reg.isValid(); 3830 } 3831 3832 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3833 const SPIRVType *ResType, 3834 MachineInstr &I) const { 3835 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3836 return selectExtInst(ResVReg, ResType, I, CL::log10); 3837 } 3838 3839 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3840 // is implemented as: 3841 // log10(x) = log2(x) * (1 / log2(10)) 3842 // = log2(x) * 0.30103 3843 3844 MachineIRBuilder MIRBuilder(I); 3845 MachineBasicBlock &BB = *I.getParent(); 3846 3847 // Build log2(x). 3848 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3849 bool Result = 3850 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3851 .addDef(VarReg) 3852 .addUse(GR.getSPIRVTypeID(ResType)) 3853 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3854 .addImm(GL::Log2) 3855 .add(I.getOperand(1)) 3856 .constrainAllUses(TII, TRI, RBI); 3857 3858 // Build 0.30103. 3859 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3860 ResType->getOpcode() == SPIRV::OpTypeFloat); 3861 // TODO: Add matrix implementation once supported by the HLSL frontend. 3862 const SPIRVType *SpirvScalarType = 3863 ResType->getOpcode() == SPIRV::OpTypeVector 3864 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3865 : ResType; 3866 Register ScaleReg = 3867 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3868 3869 // Multiply log2(x) by 0.30103 to get log10(x) result. 3870 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3871 ? SPIRV::OpVectorTimesScalar 3872 : SPIRV::OpFMulS; 3873 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3874 .addDef(ResVReg) 3875 .addUse(GR.getSPIRVTypeID(ResType)) 3876 .addUse(VarReg) 3877 .addUse(ScaleReg) 3878 .constrainAllUses(TII, TRI, RBI); 3879 } 3880 3881 // Generate the instructions to load 3-element vector builtin input 3882 // IDs/Indices. 3883 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3884 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3885 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3886 const SPIRVType *ResType, MachineInstr &I) const { 3887 MachineIRBuilder MIRBuilder(I); 3888 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3889 const SPIRVType *Vec3Ty = 3890 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3891 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3892 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3893 3894 // Create new register for the input ID builtin variable. 3895 Register NewRegister = 3896 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3897 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3898 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3899 3900 // Build global variable with the necessary decorations for the input ID 3901 // builtin variable. 3902 Register Variable = GR.buildGlobalVariable( 3903 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3904 SPIRV::StorageClass::Input, nullptr, true, true, 3905 SPIRV::LinkageType::Import, MIRBuilder, false); 3906 3907 // Create new register for loading value. 3908 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3909 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3910 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3911 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3912 3913 // Load v3uint value from the global variable. 3914 bool Result = 3915 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3916 .addDef(LoadedRegister) 3917 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3918 .addUse(Variable); 3919 3920 // Get the input ID index. Expecting operand is a constant immediate value, 3921 // wrapped in a type assignment. 3922 assert(I.getOperand(2).isReg()); 3923 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3924 3925 // Extract the input ID from the loaded vector value. 3926 MachineBasicBlock &BB = *I.getParent(); 3927 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3928 .addDef(ResVReg) 3929 .addUse(GR.getSPIRVTypeID(ResType)) 3930 .addUse(LoadedRegister) 3931 .addImm(ThreadId); 3932 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3933 } 3934 3935 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3936 MachineInstr &I) const { 3937 MachineIRBuilder MIRBuilder(I); 3938 if (Type->getOpcode() != SPIRV::OpTypeVector) 3939 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3940 3941 uint64_t VectorSize = Type->getOperand(2).getImm(); 3942 if (VectorSize == 4) 3943 return Type; 3944 3945 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3946 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3947 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3948 } 3949 3950 bool SPIRVInstructionSelector::loadHandleBeforePosition( 3951 Register &HandleReg, const SPIRVType *ResType, GIntrinsic &HandleDef, 3952 MachineInstr &Pos) const { 3953 3954 assert(HandleDef.getIntrinsicID() == 3955 Intrinsic::spv_resource_handlefrombinding); 3956 uint32_t Set = foldImm(HandleDef.getOperand(2), MRI); 3957 uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI); 3958 uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI); 3959 Register IndexReg = HandleDef.getOperand(5).getReg(); 3960 bool IsNonUniform = ArraySize > 1 && foldImm(HandleDef.getOperand(6), MRI); 3961 3962 MachineIRBuilder MIRBuilder(HandleDef); 3963 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3964 IndexReg, IsNonUniform, MIRBuilder); 3965 3966 if (IsNonUniform) 3967 buildOpDecorate(HandleReg, HandleDef, TII, SPIRV::Decoration::NonUniformEXT, 3968 {}); 3969 3970 // TODO: For now we assume the resource is an image, which needs to be 3971 // loaded to get the handle. That will not be true for storage buffers. 3972 return BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), 3973 TII.get(SPIRV::OpLoad)) 3974 .addDef(HandleReg) 3975 .addUse(GR.getSPIRVTypeID(ResType)) 3976 .addUse(VarReg) 3977 .constrainAllUses(TII, TRI, RBI); 3978 } 3979 3980 namespace llvm { 3981 InstructionSelector * 3982 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3983 const SPIRVSubtarget &Subtarget, 3984 const RegisterBankInfo &RBI) { 3985 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3986 } 3987 } // namespace llvm 3988