1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 #include "llvm/Support/ErrorHandling.h" 37 38 #define DEBUG_TYPE "spirv-isel" 39 40 using namespace llvm; 41 namespace CL = SPIRV::OpenCLExtInst; 42 namespace GL = SPIRV::GLSLExtInst; 43 44 using ExtInstList = 45 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 46 47 namespace { 48 49 llvm::SPIRV::SelectionControl::SelectionControl 50 getSelectionOperandForImm(int Imm) { 51 if (Imm == 2) 52 return SPIRV::SelectionControl::Flatten; 53 if (Imm == 1) 54 return SPIRV::SelectionControl::DontFlatten; 55 if (Imm == 0) 56 return SPIRV::SelectionControl::None; 57 llvm_unreachable("Invalid immediate"); 58 } 59 60 #define GET_GLOBALISEL_PREDICATE_BITSET 61 #include "SPIRVGenGlobalISel.inc" 62 #undef GET_GLOBALISEL_PREDICATE_BITSET 63 64 class SPIRVInstructionSelector : public InstructionSelector { 65 const SPIRVSubtarget &STI; 66 const SPIRVInstrInfo &TII; 67 const SPIRVRegisterInfo &TRI; 68 const RegisterBankInfo &RBI; 69 SPIRVGlobalRegistry &GR; 70 MachineRegisterInfo *MRI; 71 MachineFunction *HasVRegsReset = nullptr; 72 73 /// We need to keep track of the number we give to anonymous global values to 74 /// generate the same name every time when this is needed. 75 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 76 SmallPtrSet<MachineInstr *, 8> DeadMIs; 77 78 public: 79 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 80 const SPIRVSubtarget &ST, 81 const RegisterBankInfo &RBI); 82 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 83 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 84 BlockFrequencyInfo *BFI) override; 85 // Common selection code. Instruction-specific selection occurs in spvSelect. 86 bool select(MachineInstr &I) override; 87 static const char *getName() { return DEBUG_TYPE; } 88 89 #define GET_GLOBALISEL_PREDICATES_DECL 90 #include "SPIRVGenGlobalISel.inc" 91 #undef GET_GLOBALISEL_PREDICATES_DECL 92 93 #define GET_GLOBALISEL_TEMPORARIES_DECL 94 #include "SPIRVGenGlobalISel.inc" 95 #undef GET_GLOBALISEL_TEMPORARIES_DECL 96 97 private: 98 void resetVRegsType(MachineFunction &MF); 99 100 // tblgen-erated 'select' implementation, used as the initial selector for 101 // the patterns that don't require complex C++. 102 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 103 104 // All instruction-specific selection that didn't happen in "select()". 105 // Is basically a large Switch/Case delegating to all other select method. 106 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I) const; 108 109 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 110 MachineInstr &I, bool IsSigned) const; 111 112 bool selectFirstBitLow(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I) const; 114 115 bool selectFirstBitSet16(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, unsigned ExtendOpcode, 117 unsigned BitSetOpcode) const; 118 119 bool selectFirstBitSet32(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I, Register SrcReg, 121 unsigned BitSetOpcode) const; 122 123 bool selectFirstBitSet64(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I, Register SrcReg, 125 unsigned BitSetOpcode, bool SwapPrimarySide) const; 126 127 bool selectFirstBitSet64Overflow(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I, Register SrcReg, 129 unsigned BitSetOpcode, 130 bool SwapPrimarySide) const; 131 132 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 133 const MachineInstr *Init = nullptr) const; 134 135 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I, std::vector<Register> SrcRegs, 137 unsigned Opcode) const; 138 139 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 140 unsigned Opcode) const; 141 142 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 143 MachineInstr &I) const; 144 145 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 bool selectStore(MachineInstr &I) const; 148 149 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 bool selectStackRestore(MachineInstr &I) const; 152 153 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 154 155 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I, unsigned NewOpcode, 157 unsigned NegateOpcode = 0) const; 158 159 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 162 bool selectFence(MachineInstr &I) const; 163 164 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I, unsigned OpType) const; 169 170 bool selectAll(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectAny(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I) const; 181 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 185 unsigned comparisonOpcode, MachineInstr &I) const; 186 bool selectCross(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 192 MachineInstr &I) const; 193 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 194 MachineInstr &I) const; 195 196 bool selectSign(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 199 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 200 MachineInstr &I) const; 201 202 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I, unsigned Opcode) const; 204 205 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I, bool Signed) const; 207 208 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 209 MachineInstr &I) const; 210 211 template <bool Signed> 212 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 template <bool Signed> 215 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 216 MachineInstr &I) const; 217 218 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 219 int OpIdx) const; 220 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 221 int OpIdx) const; 222 223 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 224 MachineInstr &I) const; 225 226 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 227 bool IsSigned) const; 228 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 229 bool IsSigned, unsigned Opcode) const; 230 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 231 bool IsSigned) const; 232 233 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 234 MachineInstr &I) const; 235 236 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 237 bool IsSigned) const; 238 239 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 240 const SPIRVType *intTy, const SPIRVType *boolTy) const; 241 242 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 243 MachineInstr &I) const; 244 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 245 MachineInstr &I) const; 246 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 247 MachineInstr &I) const; 248 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 249 MachineInstr &I) const; 250 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 251 MachineInstr &I) const; 252 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 253 MachineInstr &I) const; 254 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I) const; 256 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 257 MachineInstr &I) const; 258 259 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 262 MachineInstr &I) const; 263 264 bool selectBranch(MachineInstr &I) const; 265 bool selectBranchCond(MachineInstr &I) const; 266 267 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 268 MachineInstr &I) const; 269 270 [[maybe_unused]] bool selectExtInst(Register ResVReg, 271 const SPIRVType *RestType, 272 MachineInstr &I, 273 GL::GLSLExtInst GLInst) const; 274 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 275 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 276 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 277 MachineInstr &I, CL::OpenCLExtInst CLInst, 278 GL::GLSLExtInst GLInst) const; 279 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 280 MachineInstr &I, const ExtInstList &ExtInsts) const; 281 282 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 283 MachineInstr &I) const; 284 285 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 286 MachineInstr &I) const; 287 288 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 289 MachineInstr &I, unsigned Opcode) const; 290 291 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 292 MachineInstr &I) const; 293 294 bool selectUnmergeValues(MachineInstr &I) const; 295 296 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 297 MachineInstr &I) const; 298 299 bool selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 300 MachineInstr &I) const; 301 302 bool selectImageWriteIntrinsic(MachineInstr &I) const; 303 304 // Utilities 305 std::pair<Register, bool> 306 buildI32Constant(uint32_t Val, MachineInstr &I, 307 const SPIRVType *ResType = nullptr) const; 308 309 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 310 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 311 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 312 MachineInstr &I) const; 313 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 314 315 bool wrapIntoSpecConstantOp(MachineInstr &I, 316 SmallVector<Register> &CompositeArgs) const; 317 318 Register getUcharPtrTypeReg(MachineInstr &I, 319 SPIRV::StorageClass::StorageClass SC) const; 320 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 321 Register Src, Register DestType, 322 uint32_t Opcode) const; 323 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 324 SPIRVType *SrcPtrTy) const; 325 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 326 uint32_t Binding, uint32_t ArraySize, 327 Register IndexReg, bool IsNonUniform, 328 MachineIRBuilder MIRBuilder) const; 329 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 330 bool extractSubvector(Register &ResVReg, const SPIRVType *ResType, 331 Register &ReadReg, MachineInstr &InsertionPoint) const; 332 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 333 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 334 Register ResVReg, const SPIRVType *ResType, 335 MachineInstr &I) const; 336 }; 337 338 } // end anonymous namespace 339 340 #define GET_GLOBALISEL_IMPL 341 #include "SPIRVGenGlobalISel.inc" 342 #undef GET_GLOBALISEL_IMPL 343 344 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 345 const SPIRVSubtarget &ST, 346 const RegisterBankInfo &RBI) 347 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 348 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 349 #define GET_GLOBALISEL_PREDICATES_INIT 350 #include "SPIRVGenGlobalISel.inc" 351 #undef GET_GLOBALISEL_PREDICATES_INIT 352 #define GET_GLOBALISEL_TEMPORARIES_INIT 353 #include "SPIRVGenGlobalISel.inc" 354 #undef GET_GLOBALISEL_TEMPORARIES_INIT 355 { 356 } 357 358 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 359 CodeGenCoverage *CoverageInfo, 360 ProfileSummaryInfo *PSI, 361 BlockFrequencyInfo *BFI) { 362 MRI = &MF.getRegInfo(); 363 GR.setCurrentFunc(MF); 364 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 365 } 366 367 // Ensure that register classes correspond to pattern matching rules. 368 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 369 if (HasVRegsReset == &MF) 370 return; 371 HasVRegsReset = &MF; 372 373 MachineRegisterInfo &MRI = MF.getRegInfo(); 374 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 375 Register Reg = Register::index2VirtReg(I); 376 LLT RegType = MRI.getType(Reg); 377 if (RegType.isScalar()) 378 MRI.setType(Reg, LLT::scalar(64)); 379 else if (RegType.isPointer()) 380 MRI.setType(Reg, LLT::pointer(0, 64)); 381 else if (RegType.isVector()) 382 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 383 } 384 for (const auto &MBB : MF) { 385 for (const auto &MI : MBB) { 386 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 387 continue; 388 Register DstReg = MI.getOperand(0).getReg(); 389 LLT DstType = MRI.getType(DstReg); 390 Register SrcReg = MI.getOperand(1).getReg(); 391 LLT SrcType = MRI.getType(SrcReg); 392 if (DstType != SrcType) 393 MRI.setType(DstReg, MRI.getType(SrcReg)); 394 395 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 396 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 397 if (DstRC != SrcRC && SrcRC) 398 MRI.setRegClass(DstReg, SrcRC); 399 } 400 } 401 } 402 403 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 404 405 // Defined in SPIRVLegalizerInfo.cpp. 406 extern bool isTypeFoldingSupported(unsigned Opcode); 407 408 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 409 for (const auto &MO : MI.all_defs()) { 410 Register Reg = MO.getReg(); 411 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 412 return false; 413 } 414 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 415 MI.isLifetimeMarker()) 416 return false; 417 if (MI.isPHI()) 418 return true; 419 if (MI.mayStore() || MI.isCall() || 420 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 421 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 422 return false; 423 return true; 424 } 425 426 bool SPIRVInstructionSelector::select(MachineInstr &I) { 427 resetVRegsType(*I.getParent()->getParent()); 428 429 assert(I.getParent() && "Instruction should be in a basic block!"); 430 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 431 432 Register Opcode = I.getOpcode(); 433 // If it's not a GMIR instruction, we've selected it already. 434 if (!isPreISelGenericOpcode(Opcode)) { 435 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 436 Register DstReg = I.getOperand(0).getReg(); 437 Register SrcReg = I.getOperand(1).getReg(); 438 auto *Def = MRI->getVRegDef(SrcReg); 439 if (isTypeFoldingSupported(Def->getOpcode())) { 440 bool Res = selectImpl(I, *CoverageInfo); 441 LLVM_DEBUG({ 442 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 443 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 444 I.print(dbgs()); 445 } 446 }); 447 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 448 if (Res) { 449 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 450 DeadMIs.insert(Def); 451 return Res; 452 } 453 } 454 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 455 MRI->replaceRegWith(SrcReg, DstReg); 456 GR.invalidateMachineInstr(&I); 457 I.removeFromParent(); 458 return true; 459 } else if (I.getNumDefs() == 1) { 460 // Make all vregs 64 bits (for SPIR-V IDs). 461 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 462 } 463 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 464 } 465 466 if (DeadMIs.contains(&I)) { 467 // if the instruction has been already made dead by folding it away 468 // erase it 469 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 470 salvageDebugInfo(*MRI, I); 471 GR.invalidateMachineInstr(&I); 472 I.eraseFromParent(); 473 return true; 474 } 475 476 if (I.getNumOperands() != I.getNumExplicitOperands()) { 477 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 478 return false; 479 } 480 481 // Common code for getting return reg+type, and removing selected instr 482 // from parent occurs here. Instr-specific selection happens in spvSelect(). 483 bool HasDefs = I.getNumDefs() > 0; 484 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 485 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 486 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 487 if (spvSelect(ResVReg, ResType, I)) { 488 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 489 for (unsigned i = 0; i < I.getNumDefs(); ++i) 490 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 491 GR.invalidateMachineInstr(&I); 492 I.removeFromParent(); 493 return true; 494 } 495 return false; 496 } 497 498 static bool mayApplyGenericSelection(unsigned Opcode) { 499 switch (Opcode) { 500 case TargetOpcode::G_CONSTANT: 501 return false; 502 case TargetOpcode::G_SADDO: 503 case TargetOpcode::G_SSUBO: 504 return true; 505 } 506 return isTypeFoldingSupported(Opcode); 507 } 508 509 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 510 MachineInstr &I) const { 511 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 512 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 513 if (DstRC != SrcRC && SrcRC) 514 MRI->setRegClass(DestReg, SrcRC); 515 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 516 TII.get(TargetOpcode::COPY)) 517 .addDef(DestReg) 518 .addUse(SrcReg) 519 .constrainAllUses(TII, TRI, RBI); 520 } 521 522 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 523 const SPIRVType *ResType, 524 MachineInstr &I) const { 525 const unsigned Opcode = I.getOpcode(); 526 if (mayApplyGenericSelection(Opcode)) 527 return selectImpl(I, *CoverageInfo); 528 switch (Opcode) { 529 case TargetOpcode::G_CONSTANT: 530 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 531 I); 532 case TargetOpcode::G_GLOBAL_VALUE: 533 return selectGlobalValue(ResVReg, I); 534 case TargetOpcode::G_IMPLICIT_DEF: 535 return selectOpUndef(ResVReg, ResType, I); 536 case TargetOpcode::G_FREEZE: 537 return selectFreeze(ResVReg, ResType, I); 538 539 case TargetOpcode::G_INTRINSIC: 540 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 541 case TargetOpcode::G_INTRINSIC_CONVERGENT: 542 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 543 return selectIntrinsic(ResVReg, ResType, I); 544 case TargetOpcode::G_BITREVERSE: 545 return selectBitreverse(ResVReg, ResType, I); 546 547 case TargetOpcode::G_BUILD_VECTOR: 548 return selectBuildVector(ResVReg, ResType, I); 549 case TargetOpcode::G_SPLAT_VECTOR: 550 return selectSplatVector(ResVReg, ResType, I); 551 552 case TargetOpcode::G_SHUFFLE_VECTOR: { 553 MachineBasicBlock &BB = *I.getParent(); 554 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 555 .addDef(ResVReg) 556 .addUse(GR.getSPIRVTypeID(ResType)) 557 .addUse(I.getOperand(1).getReg()) 558 .addUse(I.getOperand(2).getReg()); 559 for (auto V : I.getOperand(3).getShuffleMask()) 560 MIB.addImm(V); 561 return MIB.constrainAllUses(TII, TRI, RBI); 562 } 563 case TargetOpcode::G_MEMMOVE: 564 case TargetOpcode::G_MEMCPY: 565 case TargetOpcode::G_MEMSET: 566 return selectMemOperation(ResVReg, I); 567 568 case TargetOpcode::G_ICMP: 569 return selectICmp(ResVReg, ResType, I); 570 case TargetOpcode::G_FCMP: 571 return selectFCmp(ResVReg, ResType, I); 572 573 case TargetOpcode::G_FRAME_INDEX: 574 return selectFrameIndex(ResVReg, ResType, I); 575 576 case TargetOpcode::G_LOAD: 577 return selectLoad(ResVReg, ResType, I); 578 case TargetOpcode::G_STORE: 579 return selectStore(I); 580 581 case TargetOpcode::G_BR: 582 return selectBranch(I); 583 case TargetOpcode::G_BRCOND: 584 return selectBranchCond(I); 585 586 case TargetOpcode::G_PHI: 587 return selectPhi(ResVReg, ResType, I); 588 589 case TargetOpcode::G_FPTOSI: 590 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 591 case TargetOpcode::G_FPTOUI: 592 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 593 594 case TargetOpcode::G_SITOFP: 595 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 596 case TargetOpcode::G_UITOFP: 597 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 598 599 case TargetOpcode::G_CTPOP: 600 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 601 case TargetOpcode::G_SMIN: 602 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 603 case TargetOpcode::G_UMIN: 604 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 605 606 case TargetOpcode::G_SMAX: 607 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 608 case TargetOpcode::G_UMAX: 609 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 610 611 case TargetOpcode::G_SCMP: 612 return selectSUCmp(ResVReg, ResType, I, true); 613 case TargetOpcode::G_UCMP: 614 return selectSUCmp(ResVReg, ResType, I, false); 615 616 case TargetOpcode::G_STRICT_FMA: 617 case TargetOpcode::G_FMA: 618 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 619 620 case TargetOpcode::G_STRICT_FLDEXP: 621 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 622 623 case TargetOpcode::G_FPOW: 624 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 625 case TargetOpcode::G_FPOWI: 626 return selectExtInst(ResVReg, ResType, I, CL::pown); 627 628 case TargetOpcode::G_FEXP: 629 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 630 case TargetOpcode::G_FEXP2: 631 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 632 633 case TargetOpcode::G_FLOG: 634 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 635 case TargetOpcode::G_FLOG2: 636 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 637 case TargetOpcode::G_FLOG10: 638 return selectLog10(ResVReg, ResType, I); 639 640 case TargetOpcode::G_FABS: 641 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 642 case TargetOpcode::G_ABS: 643 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 644 645 case TargetOpcode::G_FMINNUM: 646 case TargetOpcode::G_FMINIMUM: 647 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 648 case TargetOpcode::G_FMAXNUM: 649 case TargetOpcode::G_FMAXIMUM: 650 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 651 652 case TargetOpcode::G_FCOPYSIGN: 653 return selectExtInst(ResVReg, ResType, I, CL::copysign); 654 655 case TargetOpcode::G_FCEIL: 656 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 657 case TargetOpcode::G_FFLOOR: 658 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 659 660 case TargetOpcode::G_FCOS: 661 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 662 case TargetOpcode::G_FSIN: 663 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 664 case TargetOpcode::G_FTAN: 665 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 666 case TargetOpcode::G_FACOS: 667 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 668 case TargetOpcode::G_FASIN: 669 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 670 case TargetOpcode::G_FATAN: 671 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 672 case TargetOpcode::G_FATAN2: 673 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 674 case TargetOpcode::G_FCOSH: 675 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 676 case TargetOpcode::G_FSINH: 677 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 678 case TargetOpcode::G_FTANH: 679 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 680 681 case TargetOpcode::G_STRICT_FSQRT: 682 case TargetOpcode::G_FSQRT: 683 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 684 685 case TargetOpcode::G_CTTZ: 686 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 687 return selectExtInst(ResVReg, ResType, I, CL::ctz); 688 case TargetOpcode::G_CTLZ: 689 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 690 return selectExtInst(ResVReg, ResType, I, CL::clz); 691 692 case TargetOpcode::G_INTRINSIC_ROUND: 693 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 694 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 695 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 696 case TargetOpcode::G_INTRINSIC_TRUNC: 697 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 698 case TargetOpcode::G_FRINT: 699 case TargetOpcode::G_FNEARBYINT: 700 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 701 702 case TargetOpcode::G_SMULH: 703 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 704 case TargetOpcode::G_UMULH: 705 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 706 707 case TargetOpcode::G_SADDSAT: 708 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 709 case TargetOpcode::G_UADDSAT: 710 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 711 case TargetOpcode::G_SSUBSAT: 712 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 713 case TargetOpcode::G_USUBSAT: 714 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 715 716 case TargetOpcode::G_UADDO: 717 return selectOverflowArith(ResVReg, ResType, I, 718 ResType->getOpcode() == SPIRV::OpTypeVector 719 ? SPIRV::OpIAddCarryV 720 : SPIRV::OpIAddCarryS); 721 case TargetOpcode::G_USUBO: 722 return selectOverflowArith(ResVReg, ResType, I, 723 ResType->getOpcode() == SPIRV::OpTypeVector 724 ? SPIRV::OpISubBorrowV 725 : SPIRV::OpISubBorrowS); 726 case TargetOpcode::G_UMULO: 727 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 728 case TargetOpcode::G_SMULO: 729 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 730 731 case TargetOpcode::G_SEXT: 732 return selectExt(ResVReg, ResType, I, true); 733 case TargetOpcode::G_ANYEXT: 734 case TargetOpcode::G_ZEXT: 735 return selectExt(ResVReg, ResType, I, false); 736 case TargetOpcode::G_TRUNC: 737 return selectTrunc(ResVReg, ResType, I); 738 case TargetOpcode::G_FPTRUNC: 739 case TargetOpcode::G_FPEXT: 740 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 741 742 case TargetOpcode::G_PTRTOINT: 743 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 744 case TargetOpcode::G_INTTOPTR: 745 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 746 case TargetOpcode::G_BITCAST: 747 return selectBitcast(ResVReg, ResType, I); 748 case TargetOpcode::G_ADDRSPACE_CAST: 749 return selectAddrSpaceCast(ResVReg, ResType, I); 750 case TargetOpcode::G_PTR_ADD: { 751 // Currently, we get G_PTR_ADD only applied to global variables. 752 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 753 Register GV = I.getOperand(1).getReg(); 754 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 755 (void)II; 756 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 757 (*II).getOpcode() == TargetOpcode::COPY || 758 (*II).getOpcode() == SPIRV::OpVariable) && 759 isImm(I.getOperand(2), MRI)); 760 // It may be the initialization of a global variable. 761 bool IsGVInit = false; 762 for (MachineRegisterInfo::use_instr_iterator 763 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 764 UseEnd = MRI->use_instr_end(); 765 UseIt != UseEnd; UseIt = std::next(UseIt)) { 766 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 767 (*UseIt).getOpcode() == SPIRV::OpVariable) { 768 IsGVInit = true; 769 break; 770 } 771 } 772 MachineBasicBlock &BB = *I.getParent(); 773 if (!IsGVInit) { 774 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 775 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 776 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 777 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 778 // Build a new virtual register that is associated with the required 779 // data type. 780 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 781 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 782 // Having a correctly typed base we are ready to build the actually 783 // required GEP. It may not be a constant though, because all Operands 784 // of OpSpecConstantOp is to originate from other const instructions, 785 // and only the AccessChain named opcodes accept a global OpVariable 786 // instruction. We can't use an AccessChain opcode because of the type 787 // mismatch between result and base types. 788 if (!GR.isBitcastCompatible(ResType, GVType)) 789 report_fatal_error( 790 "incompatible result and operand types in a bitcast"); 791 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 792 MachineInstrBuilder MIB = 793 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 794 .addDef(NewVReg) 795 .addUse(ResTypeReg) 796 .addUse(GV); 797 return MIB.constrainAllUses(TII, TRI, RBI) && 798 BuildMI(BB, I, I.getDebugLoc(), 799 TII.get(STI.isVulkanEnv() 800 ? SPIRV::OpInBoundsAccessChain 801 : SPIRV::OpInBoundsPtrAccessChain)) 802 .addDef(ResVReg) 803 .addUse(ResTypeReg) 804 .addUse(NewVReg) 805 .addUse(I.getOperand(2).getReg()) 806 .constrainAllUses(TII, TRI, RBI); 807 } else { 808 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 809 .addDef(ResVReg) 810 .addUse(GR.getSPIRVTypeID(ResType)) 811 .addImm( 812 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 813 .addUse(GV) 814 .addUse(I.getOperand(2).getReg()) 815 .constrainAllUses(TII, TRI, RBI); 816 } 817 } 818 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 819 // initialize a global variable with a constant expression (e.g., the test 820 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 821 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 822 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 823 .addDef(ResVReg) 824 .addUse(GR.getSPIRVTypeID(ResType)) 825 .addImm(static_cast<uint32_t>( 826 SPIRV::Opcode::InBoundsPtrAccessChain)) 827 .addUse(GV) 828 .addUse(Idx) 829 .addUse(I.getOperand(2).getReg()); 830 return MIB.constrainAllUses(TII, TRI, RBI); 831 } 832 833 case TargetOpcode::G_ATOMICRMW_OR: 834 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 835 case TargetOpcode::G_ATOMICRMW_ADD: 836 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 837 case TargetOpcode::G_ATOMICRMW_AND: 838 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 839 case TargetOpcode::G_ATOMICRMW_MAX: 840 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 841 case TargetOpcode::G_ATOMICRMW_MIN: 842 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 843 case TargetOpcode::G_ATOMICRMW_SUB: 844 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 845 case TargetOpcode::G_ATOMICRMW_XOR: 846 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 847 case TargetOpcode::G_ATOMICRMW_UMAX: 848 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 849 case TargetOpcode::G_ATOMICRMW_UMIN: 850 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 851 case TargetOpcode::G_ATOMICRMW_XCHG: 852 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 853 case TargetOpcode::G_ATOMIC_CMPXCHG: 854 return selectAtomicCmpXchg(ResVReg, ResType, I); 855 856 case TargetOpcode::G_ATOMICRMW_FADD: 857 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 858 case TargetOpcode::G_ATOMICRMW_FSUB: 859 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 860 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 861 SPIRV::OpFNegate); 862 case TargetOpcode::G_ATOMICRMW_FMIN: 863 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 864 case TargetOpcode::G_ATOMICRMW_FMAX: 865 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 866 867 case TargetOpcode::G_FENCE: 868 return selectFence(I); 869 870 case TargetOpcode::G_STACKSAVE: 871 return selectStackSave(ResVReg, ResType, I); 872 case TargetOpcode::G_STACKRESTORE: 873 return selectStackRestore(I); 874 875 case TargetOpcode::G_UNMERGE_VALUES: 876 return selectUnmergeValues(I); 877 878 // Discard gen opcodes for intrinsics which we do not expect to actually 879 // represent code after lowering or intrinsics which are not implemented but 880 // should not crash when found in a customer's LLVM IR input. 881 case TargetOpcode::G_TRAP: 882 case TargetOpcode::G_DEBUGTRAP: 883 case TargetOpcode::G_UBSANTRAP: 884 case TargetOpcode::DBG_LABEL: 885 return true; 886 887 default: 888 return false; 889 } 890 } 891 892 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 893 const SPIRVType *ResType, 894 MachineInstr &I, 895 GL::GLSLExtInst GLInst) const { 896 return selectExtInst(ResVReg, ResType, I, 897 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 898 } 899 900 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 901 const SPIRVType *ResType, 902 MachineInstr &I, 903 CL::OpenCLExtInst CLInst) const { 904 return selectExtInst(ResVReg, ResType, I, 905 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 906 } 907 908 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 909 const SPIRVType *ResType, 910 MachineInstr &I, 911 CL::OpenCLExtInst CLInst, 912 GL::GLSLExtInst GLInst) const { 913 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 914 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 915 return selectExtInst(ResVReg, ResType, I, ExtInsts); 916 } 917 918 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 919 const SPIRVType *ResType, 920 MachineInstr &I, 921 const ExtInstList &Insts) const { 922 923 for (const auto &Ex : Insts) { 924 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 925 uint32_t Opcode = Ex.second; 926 if (STI.canUseExtInstSet(Set)) { 927 MachineBasicBlock &BB = *I.getParent(); 928 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 929 .addDef(ResVReg) 930 .addUse(GR.getSPIRVTypeID(ResType)) 931 .addImm(static_cast<uint32_t>(Set)) 932 .addImm(Opcode); 933 const unsigned NumOps = I.getNumOperands(); 934 unsigned Index = 1; 935 if (Index < NumOps && 936 I.getOperand(Index).getType() == 937 MachineOperand::MachineOperandType::MO_IntrinsicID) 938 Index = 2; 939 for (; Index < NumOps; ++Index) 940 MIB.add(I.getOperand(Index)); 941 return MIB.constrainAllUses(TII, TRI, RBI); 942 } 943 } 944 return false; 945 } 946 947 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 948 const SPIRVType *ResType, 949 MachineInstr &I, 950 std::vector<Register> Srcs, 951 unsigned Opcode) const { 952 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 953 .addDef(ResVReg) 954 .addUse(GR.getSPIRVTypeID(ResType)); 955 for (Register SReg : Srcs) { 956 MIB.addUse(SReg); 957 } 958 return MIB.constrainAllUses(TII, TRI, RBI); 959 } 960 961 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 962 const SPIRVType *ResType, 963 MachineInstr &I, 964 unsigned Opcode) const { 965 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 966 Register SrcReg = I.getOperand(1).getReg(); 967 bool IsGV = false; 968 for (MachineRegisterInfo::def_instr_iterator DefIt = 969 MRI->def_instr_begin(SrcReg); 970 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 971 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 972 IsGV = true; 973 break; 974 } 975 } 976 if (IsGV) { 977 uint32_t SpecOpcode = 0; 978 switch (Opcode) { 979 case SPIRV::OpConvertPtrToU: 980 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 981 break; 982 case SPIRV::OpConvertUToPtr: 983 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 984 break; 985 } 986 if (SpecOpcode) 987 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 988 TII.get(SPIRV::OpSpecConstantOp)) 989 .addDef(ResVReg) 990 .addUse(GR.getSPIRVTypeID(ResType)) 991 .addImm(SpecOpcode) 992 .addUse(SrcReg) 993 .constrainAllUses(TII, TRI, RBI); 994 } 995 } 996 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 997 Opcode); 998 } 999 1000 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 1001 const SPIRVType *ResType, 1002 MachineInstr &I) const { 1003 Register OpReg = I.getOperand(1).getReg(); 1004 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 1005 if (!GR.isBitcastCompatible(ResType, OpType)) 1006 report_fatal_error("incompatible result and operand types in a bitcast"); 1007 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1008 } 1009 1010 static void addMemoryOperands(MachineMemOperand *MemOp, 1011 MachineInstrBuilder &MIB) { 1012 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1013 if (MemOp->isVolatile()) 1014 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1015 if (MemOp->isNonTemporal()) 1016 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1017 if (MemOp->getAlign().value()) 1018 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 1019 1020 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 1021 MIB.addImm(SpvMemOp); 1022 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 1023 MIB.addImm(MemOp->getAlign().value()); 1024 } 1025 } 1026 1027 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1028 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1029 if (Flags & MachineMemOperand::Flags::MOVolatile) 1030 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1031 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1032 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1033 1034 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1035 MIB.addImm(SpvMemOp); 1036 } 1037 1038 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1039 const SPIRVType *ResType, 1040 MachineInstr &I) const { 1041 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1042 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1043 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1044 .addDef(ResVReg) 1045 .addUse(GR.getSPIRVTypeID(ResType)) 1046 .addUse(Ptr); 1047 if (!I.getNumMemOperands()) { 1048 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1049 I.getOpcode() == 1050 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1051 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1052 } else { 1053 addMemoryOperands(*I.memoperands_begin(), MIB); 1054 } 1055 return MIB.constrainAllUses(TII, TRI, RBI); 1056 } 1057 1058 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1059 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1060 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1061 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1062 MachineBasicBlock &BB = *I.getParent(); 1063 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1064 .addUse(Ptr) 1065 .addUse(StoreVal); 1066 if (!I.getNumMemOperands()) { 1067 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1068 I.getOpcode() == 1069 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1070 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1071 } else { 1072 addMemoryOperands(*I.memoperands_begin(), MIB); 1073 } 1074 return MIB.constrainAllUses(TII, TRI, RBI); 1075 } 1076 1077 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1078 const SPIRVType *ResType, 1079 MachineInstr &I) const { 1080 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1081 report_fatal_error( 1082 "llvm.stacksave intrinsic: this instruction requires the following " 1083 "SPIR-V extension: SPV_INTEL_variable_length_array", 1084 false); 1085 MachineBasicBlock &BB = *I.getParent(); 1086 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1087 .addDef(ResVReg) 1088 .addUse(GR.getSPIRVTypeID(ResType)) 1089 .constrainAllUses(TII, TRI, RBI); 1090 } 1091 1092 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1093 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1094 report_fatal_error( 1095 "llvm.stackrestore intrinsic: this instruction requires the following " 1096 "SPIR-V extension: SPV_INTEL_variable_length_array", 1097 false); 1098 if (!I.getOperand(0).isReg()) 1099 return false; 1100 MachineBasicBlock &BB = *I.getParent(); 1101 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1102 .addUse(I.getOperand(0).getReg()) 1103 .constrainAllUses(TII, TRI, RBI); 1104 } 1105 1106 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1107 MachineInstr &I) const { 1108 MachineBasicBlock &BB = *I.getParent(); 1109 Register SrcReg = I.getOperand(1).getReg(); 1110 bool Result = true; 1111 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1112 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1113 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1114 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1115 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1116 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1117 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1118 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1119 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1120 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1121 Function &CurFunction = GR.CurMF->getFunction(); 1122 Type *LLVMArrTy = 1123 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1124 // Module takes ownership of the global var. 1125 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1126 true, GlobalValue::InternalLinkage, 1127 Constant::getNullValue(LLVMArrTy)); 1128 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1129 GR.add(GV, GR.CurMF, VarReg); 1130 GR.addGlobalObject(GV, GR.CurMF, VarReg); 1131 1132 Result &= 1133 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1134 .addDef(VarReg) 1135 .addUse(GR.getSPIRVTypeID(VarTy)) 1136 .addImm(SPIRV::StorageClass::UniformConstant) 1137 .addUse(Const) 1138 .constrainAllUses(TII, TRI, RBI); 1139 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1140 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1141 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1142 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1143 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1144 } 1145 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1146 .addUse(I.getOperand(0).getReg()) 1147 .addUse(SrcReg) 1148 .addUse(I.getOperand(2).getReg()); 1149 if (I.getNumMemOperands()) 1150 addMemoryOperands(*I.memoperands_begin(), MIB); 1151 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1152 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1153 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1154 return Result; 1155 } 1156 1157 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1158 const SPIRVType *ResType, 1159 MachineInstr &I, 1160 unsigned NewOpcode, 1161 unsigned NegateOpcode) const { 1162 bool Result = true; 1163 assert(I.hasOneMemOperand()); 1164 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1165 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1166 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1167 auto ScopeConstant = buildI32Constant(Scope, I); 1168 Register ScopeReg = ScopeConstant.first; 1169 Result &= ScopeConstant.second; 1170 1171 Register Ptr = I.getOperand(1).getReg(); 1172 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1173 // auto ScSem = 1174 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1175 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1176 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1177 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1178 Register MemSemReg = MemSemConstant.first; 1179 Result &= MemSemConstant.second; 1180 1181 Register ValueReg = I.getOperand(2).getReg(); 1182 if (NegateOpcode != 0) { 1183 // Translation with negative value operand is requested 1184 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1185 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1186 ValueReg = TmpReg; 1187 } 1188 1189 return Result && 1190 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1191 .addDef(ResVReg) 1192 .addUse(GR.getSPIRVTypeID(ResType)) 1193 .addUse(Ptr) 1194 .addUse(ScopeReg) 1195 .addUse(MemSemReg) 1196 .addUse(ValueReg) 1197 .constrainAllUses(TII, TRI, RBI); 1198 } 1199 1200 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1201 unsigned ArgI = I.getNumOperands() - 1; 1202 Register SrcReg = 1203 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1204 SPIRVType *DefType = 1205 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1206 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1207 report_fatal_error( 1208 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1209 1210 SPIRVType *ScalarType = 1211 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1212 MachineBasicBlock &BB = *I.getParent(); 1213 bool Res = false; 1214 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1215 Register ResVReg = I.getOperand(i).getReg(); 1216 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1217 if (!ResType) { 1218 // There was no "assign type" actions, let's fix this now 1219 ResType = ScalarType; 1220 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1221 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1222 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1223 } 1224 auto MIB = 1225 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1226 .addDef(ResVReg) 1227 .addUse(GR.getSPIRVTypeID(ResType)) 1228 .addUse(SrcReg) 1229 .addImm(static_cast<int64_t>(i)); 1230 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1231 } 1232 return Res; 1233 } 1234 1235 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1236 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1237 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1238 auto MemSemConstant = buildI32Constant(MemSem, I); 1239 Register MemSemReg = MemSemConstant.first; 1240 bool Result = MemSemConstant.second; 1241 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1242 uint32_t Scope = static_cast<uint32_t>( 1243 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1244 auto ScopeConstant = buildI32Constant(Scope, I); 1245 Register ScopeReg = ScopeConstant.first; 1246 Result &= ScopeConstant.second; 1247 MachineBasicBlock &BB = *I.getParent(); 1248 return Result && 1249 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1250 .addUse(ScopeReg) 1251 .addUse(MemSemReg) 1252 .constrainAllUses(TII, TRI, RBI); 1253 } 1254 1255 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1256 const SPIRVType *ResType, 1257 MachineInstr &I, 1258 unsigned Opcode) const { 1259 Type *ResTy = nullptr; 1260 StringRef ResName; 1261 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1262 report_fatal_error( 1263 "Not enough info to select the arithmetic with overflow instruction"); 1264 if (!ResTy || !ResTy->isStructTy()) 1265 report_fatal_error("Expect struct type result for the arithmetic " 1266 "with overflow instruction"); 1267 // "Result Type must be from OpTypeStruct. The struct must have two members, 1268 // and the two members must be the same type." 1269 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1270 ResTy = StructType::get(ResElemTy, ResElemTy); 1271 // Build SPIR-V types and constant(s) if needed. 1272 MachineIRBuilder MIRBuilder(I); 1273 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1274 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1275 assert(I.getNumDefs() > 1 && "Not enought operands"); 1276 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1277 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1278 if (N > 1) 1279 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1280 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1281 Register ZeroReg = buildZerosVal(ResType, I); 1282 // A new virtual register to store the result struct. 1283 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1284 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1285 // Build the result name if needed. 1286 if (ResName.size() > 0) 1287 buildOpName(StructVReg, ResName, MIRBuilder); 1288 // Build the arithmetic with overflow instruction. 1289 MachineBasicBlock &BB = *I.getParent(); 1290 auto MIB = 1291 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1292 .addDef(StructVReg) 1293 .addUse(GR.getSPIRVTypeID(StructType)); 1294 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1295 MIB.addUse(I.getOperand(i).getReg()); 1296 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1297 // Build instructions to extract fields of the instruction's result. 1298 // A new virtual register to store the higher part of the result struct. 1299 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1300 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1301 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1302 auto MIB = 1303 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1304 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1305 .addUse(GR.getSPIRVTypeID(ResType)) 1306 .addUse(StructVReg) 1307 .addImm(i); 1308 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1309 } 1310 // Build boolean value from the higher part. 1311 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1312 .addDef(I.getOperand(1).getReg()) 1313 .addUse(BoolTypeReg) 1314 .addUse(HigherVReg) 1315 .addUse(ZeroReg) 1316 .constrainAllUses(TII, TRI, RBI); 1317 } 1318 1319 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1320 const SPIRVType *ResType, 1321 MachineInstr &I) const { 1322 bool Result = true; 1323 Register ScopeReg; 1324 Register MemSemEqReg; 1325 Register MemSemNeqReg; 1326 Register Ptr = I.getOperand(2).getReg(); 1327 if (!isa<GIntrinsic>(I)) { 1328 assert(I.hasOneMemOperand()); 1329 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1330 unsigned Scope = static_cast<uint32_t>(getMemScope( 1331 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1332 auto ScopeConstant = buildI32Constant(Scope, I); 1333 ScopeReg = ScopeConstant.first; 1334 Result &= ScopeConstant.second; 1335 1336 unsigned ScSem = static_cast<uint32_t>( 1337 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1338 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1339 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1340 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1341 MemSemEqReg = MemSemEqConstant.first; 1342 Result &= MemSemEqConstant.second; 1343 AtomicOrdering FO = MemOp->getFailureOrdering(); 1344 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1345 if (MemSemEq == MemSemNeq) 1346 MemSemNeqReg = MemSemEqReg; 1347 else { 1348 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1349 MemSemNeqReg = MemSemNeqConstant.first; 1350 Result &= MemSemNeqConstant.second; 1351 } 1352 } else { 1353 ScopeReg = I.getOperand(5).getReg(); 1354 MemSemEqReg = I.getOperand(6).getReg(); 1355 MemSemNeqReg = I.getOperand(7).getReg(); 1356 } 1357 1358 Register Cmp = I.getOperand(3).getReg(); 1359 Register Val = I.getOperand(4).getReg(); 1360 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1361 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1362 const DebugLoc &DL = I.getDebugLoc(); 1363 Result &= 1364 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1365 .addDef(ACmpRes) 1366 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1367 .addUse(Ptr) 1368 .addUse(ScopeReg) 1369 .addUse(MemSemEqReg) 1370 .addUse(MemSemNeqReg) 1371 .addUse(Val) 1372 .addUse(Cmp) 1373 .constrainAllUses(TII, TRI, RBI); 1374 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1375 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1376 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1377 .addDef(CmpSuccReg) 1378 .addUse(GR.getSPIRVTypeID(BoolTy)) 1379 .addUse(ACmpRes) 1380 .addUse(Cmp) 1381 .constrainAllUses(TII, TRI, RBI); 1382 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1383 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1384 .addDef(TmpReg) 1385 .addUse(GR.getSPIRVTypeID(ResType)) 1386 .addUse(ACmpRes) 1387 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1388 .addImm(0) 1389 .constrainAllUses(TII, TRI, RBI); 1390 return Result && 1391 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1392 .addDef(ResVReg) 1393 .addUse(GR.getSPIRVTypeID(ResType)) 1394 .addUse(CmpSuccReg) 1395 .addUse(TmpReg) 1396 .addImm(1) 1397 .constrainAllUses(TII, TRI, RBI); 1398 } 1399 1400 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1401 switch (SC) { 1402 case SPIRV::StorageClass::Workgroup: 1403 case SPIRV::StorageClass::CrossWorkgroup: 1404 case SPIRV::StorageClass::Function: 1405 return true; 1406 default: 1407 return false; 1408 } 1409 } 1410 1411 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1412 switch (SC) { 1413 case SPIRV::StorageClass::DeviceOnlyINTEL: 1414 case SPIRV::StorageClass::HostOnlyINTEL: 1415 return true; 1416 default: 1417 return false; 1418 } 1419 } 1420 1421 // Returns true ResVReg is referred only from global vars and OpName's. 1422 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1423 bool IsGRef = false; 1424 bool IsAllowedRefs = 1425 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1426 [&IsGRef](auto const &It) { 1427 unsigned Opcode = It.getOpcode(); 1428 if (Opcode == SPIRV::OpConstantComposite || 1429 Opcode == SPIRV::OpVariable || 1430 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1431 return IsGRef = true; 1432 return Opcode == SPIRV::OpName; 1433 }); 1434 return IsAllowedRefs && IsGRef; 1435 } 1436 1437 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1438 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1439 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1440 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1441 } 1442 1443 MachineInstrBuilder 1444 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1445 Register Src, Register DestType, 1446 uint32_t Opcode) const { 1447 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1448 TII.get(SPIRV::OpSpecConstantOp)) 1449 .addDef(Dest) 1450 .addUse(DestType) 1451 .addImm(Opcode) 1452 .addUse(Src); 1453 } 1454 1455 MachineInstrBuilder 1456 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1457 SPIRVType *SrcPtrTy) const { 1458 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1459 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1460 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1461 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1462 SPIRV::StorageClass::Generic), 1463 GR.getPointerSize())); 1464 MachineFunction *MF = I.getParent()->getParent(); 1465 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1466 MachineInstrBuilder MIB = buildSpecConstantOp( 1467 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1468 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1469 GR.add(MIB.getInstr(), MF, Tmp); 1470 return MIB; 1471 } 1472 1473 // In SPIR-V address space casting can only happen to and from the Generic 1474 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1475 // pointers to and from Generic pointers. As such, we can convert e.g. from 1476 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1477 // other combinations can only be done by a bitcast, and are probably not safe. 1478 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1479 const SPIRVType *ResType, 1480 MachineInstr &I) const { 1481 MachineBasicBlock &BB = *I.getParent(); 1482 const DebugLoc &DL = I.getDebugLoc(); 1483 1484 Register SrcPtr = I.getOperand(1).getReg(); 1485 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1486 1487 // don't generate a cast for a null that may be represented by OpTypeInt 1488 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1489 ResType->getOpcode() != SPIRV::OpTypePointer) 1490 return BuildCOPY(ResVReg, SrcPtr, I); 1491 1492 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1493 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1494 1495 if (isASCastInGVar(MRI, ResVReg)) { 1496 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1497 // are expressed by OpSpecConstantOp with an Opcode. 1498 // TODO: maybe insert a check whether the Kernel capability was declared and 1499 // so PtrCastToGeneric/GenericCastToPtr are available. 1500 unsigned SpecOpcode = 1501 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1502 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1503 : (SrcSC == SPIRV::StorageClass::Generic && 1504 isGenericCastablePtr(DstSC) 1505 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1506 : 0); 1507 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1508 // correct value of ResType and use general i8* instead. Maybe this should 1509 // be addressed in the emit-intrinsic step to infer a correct 1510 // OpConstantComposite type. 1511 if (SpecOpcode) { 1512 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1513 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1514 .constrainAllUses(TII, TRI, RBI); 1515 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1516 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1517 return MIB.constrainAllUses(TII, TRI, RBI) && 1518 buildSpecConstantOp( 1519 I, ResVReg, MIB->getOperand(0).getReg(), 1520 getUcharPtrTypeReg(I, DstSC), 1521 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1522 .constrainAllUses(TII, TRI, RBI); 1523 } 1524 } 1525 1526 // don't generate a cast between identical storage classes 1527 if (SrcSC == DstSC) 1528 return BuildCOPY(ResVReg, SrcPtr, I); 1529 1530 if ((SrcSC == SPIRV::StorageClass::Function && 1531 DstSC == SPIRV::StorageClass::Private) || 1532 (DstSC == SPIRV::StorageClass::Function && 1533 SrcSC == SPIRV::StorageClass::Private)) 1534 return BuildCOPY(ResVReg, SrcPtr, I); 1535 1536 // Casting from an eligible pointer to Generic. 1537 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1538 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1539 // Casting from Generic to an eligible pointer. 1540 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1541 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1542 // Casting between 2 eligible pointers using Generic as an intermediary. 1543 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1544 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1545 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1546 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1547 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1548 .addDef(Tmp) 1549 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1550 .addUse(SrcPtr) 1551 .constrainAllUses(TII, TRI, RBI); 1552 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1553 .addDef(ResVReg) 1554 .addUse(GR.getSPIRVTypeID(ResType)) 1555 .addUse(Tmp) 1556 .constrainAllUses(TII, TRI, RBI); 1557 } 1558 1559 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1560 // be applied 1561 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1562 return selectUnOp(ResVReg, ResType, I, 1563 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1564 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1565 return selectUnOp(ResVReg, ResType, I, 1566 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1567 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1568 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1569 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1570 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1571 1572 // Bitcast for pointers requires that the address spaces must match 1573 return false; 1574 } 1575 1576 static unsigned getFCmpOpcode(unsigned PredNum) { 1577 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1578 switch (Pred) { 1579 case CmpInst::FCMP_OEQ: 1580 return SPIRV::OpFOrdEqual; 1581 case CmpInst::FCMP_OGE: 1582 return SPIRV::OpFOrdGreaterThanEqual; 1583 case CmpInst::FCMP_OGT: 1584 return SPIRV::OpFOrdGreaterThan; 1585 case CmpInst::FCMP_OLE: 1586 return SPIRV::OpFOrdLessThanEqual; 1587 case CmpInst::FCMP_OLT: 1588 return SPIRV::OpFOrdLessThan; 1589 case CmpInst::FCMP_ONE: 1590 return SPIRV::OpFOrdNotEqual; 1591 case CmpInst::FCMP_ORD: 1592 return SPIRV::OpOrdered; 1593 case CmpInst::FCMP_UEQ: 1594 return SPIRV::OpFUnordEqual; 1595 case CmpInst::FCMP_UGE: 1596 return SPIRV::OpFUnordGreaterThanEqual; 1597 case CmpInst::FCMP_UGT: 1598 return SPIRV::OpFUnordGreaterThan; 1599 case CmpInst::FCMP_ULE: 1600 return SPIRV::OpFUnordLessThanEqual; 1601 case CmpInst::FCMP_ULT: 1602 return SPIRV::OpFUnordLessThan; 1603 case CmpInst::FCMP_UNE: 1604 return SPIRV::OpFUnordNotEqual; 1605 case CmpInst::FCMP_UNO: 1606 return SPIRV::OpUnordered; 1607 default: 1608 llvm_unreachable("Unknown predicate type for FCmp"); 1609 } 1610 } 1611 1612 static unsigned getICmpOpcode(unsigned PredNum) { 1613 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1614 switch (Pred) { 1615 case CmpInst::ICMP_EQ: 1616 return SPIRV::OpIEqual; 1617 case CmpInst::ICMP_NE: 1618 return SPIRV::OpINotEqual; 1619 case CmpInst::ICMP_SGE: 1620 return SPIRV::OpSGreaterThanEqual; 1621 case CmpInst::ICMP_SGT: 1622 return SPIRV::OpSGreaterThan; 1623 case CmpInst::ICMP_SLE: 1624 return SPIRV::OpSLessThanEqual; 1625 case CmpInst::ICMP_SLT: 1626 return SPIRV::OpSLessThan; 1627 case CmpInst::ICMP_UGE: 1628 return SPIRV::OpUGreaterThanEqual; 1629 case CmpInst::ICMP_UGT: 1630 return SPIRV::OpUGreaterThan; 1631 case CmpInst::ICMP_ULE: 1632 return SPIRV::OpULessThanEqual; 1633 case CmpInst::ICMP_ULT: 1634 return SPIRV::OpULessThan; 1635 default: 1636 llvm_unreachable("Unknown predicate type for ICmp"); 1637 } 1638 } 1639 1640 static unsigned getPtrCmpOpcode(unsigned Pred) { 1641 switch (static_cast<CmpInst::Predicate>(Pred)) { 1642 case CmpInst::ICMP_EQ: 1643 return SPIRV::OpPtrEqual; 1644 case CmpInst::ICMP_NE: 1645 return SPIRV::OpPtrNotEqual; 1646 default: 1647 llvm_unreachable("Unknown predicate type for pointer comparison"); 1648 } 1649 } 1650 1651 // Return the logical operation, or abort if none exists. 1652 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1653 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1654 switch (Pred) { 1655 case CmpInst::ICMP_EQ: 1656 return SPIRV::OpLogicalEqual; 1657 case CmpInst::ICMP_NE: 1658 return SPIRV::OpLogicalNotEqual; 1659 default: 1660 llvm_unreachable("Unknown predicate type for Bool comparison"); 1661 } 1662 } 1663 1664 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1665 if (!LLVMFloatTy) 1666 return APFloat::getZero(APFloat::IEEEsingle()); 1667 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1668 case Type::HalfTyID: 1669 return APFloat::getZero(APFloat::IEEEhalf()); 1670 default: 1671 case Type::FloatTyID: 1672 return APFloat::getZero(APFloat::IEEEsingle()); 1673 case Type::DoubleTyID: 1674 return APFloat::getZero(APFloat::IEEEdouble()); 1675 } 1676 } 1677 1678 static APFloat getOneFP(const Type *LLVMFloatTy) { 1679 if (!LLVMFloatTy) 1680 return APFloat::getOne(APFloat::IEEEsingle()); 1681 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1682 case Type::HalfTyID: 1683 return APFloat::getOne(APFloat::IEEEhalf()); 1684 default: 1685 case Type::FloatTyID: 1686 return APFloat::getOne(APFloat::IEEEsingle()); 1687 case Type::DoubleTyID: 1688 return APFloat::getOne(APFloat::IEEEdouble()); 1689 } 1690 } 1691 1692 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1693 const SPIRVType *ResType, 1694 MachineInstr &I, 1695 unsigned OpAnyOrAll) const { 1696 assert(I.getNumOperands() == 3); 1697 assert(I.getOperand(2).isReg()); 1698 MachineBasicBlock &BB = *I.getParent(); 1699 Register InputRegister = I.getOperand(2).getReg(); 1700 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1701 1702 if (!InputType) 1703 report_fatal_error("Input Type could not be determined."); 1704 1705 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1706 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1707 if (IsBoolTy && !IsVectorTy) { 1708 assert(ResVReg == I.getOperand(0).getReg()); 1709 return BuildCOPY(ResVReg, InputRegister, I); 1710 } 1711 1712 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1713 unsigned SpirvNotEqualId = 1714 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1715 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1716 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1717 Register NotEqualReg = ResVReg; 1718 1719 if (IsVectorTy) { 1720 NotEqualReg = IsBoolTy ? InputRegister 1721 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1722 const unsigned NumElts = InputType->getOperand(2).getImm(); 1723 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1724 } 1725 1726 bool Result = true; 1727 if (!IsBoolTy) { 1728 Register ConstZeroReg = 1729 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1730 1731 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1732 .addDef(NotEqualReg) 1733 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1734 .addUse(InputRegister) 1735 .addUse(ConstZeroReg) 1736 .constrainAllUses(TII, TRI, RBI); 1737 } 1738 1739 if (!IsVectorTy) 1740 return Result; 1741 1742 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1743 .addDef(ResVReg) 1744 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1745 .addUse(NotEqualReg) 1746 .constrainAllUses(TII, TRI, RBI); 1747 } 1748 1749 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1750 const SPIRVType *ResType, 1751 MachineInstr &I) const { 1752 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1753 } 1754 1755 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1756 const SPIRVType *ResType, 1757 MachineInstr &I) const { 1758 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1759 } 1760 1761 // Select the OpDot instruction for the given float dot 1762 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1763 const SPIRVType *ResType, 1764 MachineInstr &I) const { 1765 assert(I.getNumOperands() == 4); 1766 assert(I.getOperand(2).isReg()); 1767 assert(I.getOperand(3).isReg()); 1768 1769 [[maybe_unused]] SPIRVType *VecType = 1770 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1771 1772 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1773 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1774 "dot product requires a vector of at least 2 components"); 1775 1776 [[maybe_unused]] SPIRVType *EltType = 1777 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1778 1779 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1780 1781 MachineBasicBlock &BB = *I.getParent(); 1782 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1783 .addDef(ResVReg) 1784 .addUse(GR.getSPIRVTypeID(ResType)) 1785 .addUse(I.getOperand(2).getReg()) 1786 .addUse(I.getOperand(3).getReg()) 1787 .constrainAllUses(TII, TRI, RBI); 1788 } 1789 1790 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1791 const SPIRVType *ResType, 1792 MachineInstr &I, 1793 bool Signed) const { 1794 assert(I.getNumOperands() == 4); 1795 assert(I.getOperand(2).isReg()); 1796 assert(I.getOperand(3).isReg()); 1797 MachineBasicBlock &BB = *I.getParent(); 1798 1799 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1800 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1801 .addDef(ResVReg) 1802 .addUse(GR.getSPIRVTypeID(ResType)) 1803 .addUse(I.getOperand(2).getReg()) 1804 .addUse(I.getOperand(3).getReg()) 1805 .constrainAllUses(TII, TRI, RBI); 1806 } 1807 1808 // Since pre-1.6 SPIRV has no integer dot implementation, 1809 // expand by piecewise multiplying and adding the results 1810 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1811 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1812 assert(I.getNumOperands() == 4); 1813 assert(I.getOperand(2).isReg()); 1814 assert(I.getOperand(3).isReg()); 1815 MachineBasicBlock &BB = *I.getParent(); 1816 1817 // Multiply the vectors, then sum the results 1818 Register Vec0 = I.getOperand(2).getReg(); 1819 Register Vec1 = I.getOperand(3).getReg(); 1820 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1821 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1822 1823 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1824 .addDef(TmpVec) 1825 .addUse(GR.getSPIRVTypeID(VecType)) 1826 .addUse(Vec0) 1827 .addUse(Vec1) 1828 .constrainAllUses(TII, TRI, RBI); 1829 1830 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1831 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1832 "dot product requires a vector of at least 2 components"); 1833 1834 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1835 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1836 .addDef(Res) 1837 .addUse(GR.getSPIRVTypeID(ResType)) 1838 .addUse(TmpVec) 1839 .addImm(0) 1840 .constrainAllUses(TII, TRI, RBI); 1841 1842 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1843 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1844 1845 Result &= 1846 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1847 .addDef(Elt) 1848 .addUse(GR.getSPIRVTypeID(ResType)) 1849 .addUse(TmpVec) 1850 .addImm(i) 1851 .constrainAllUses(TII, TRI, RBI); 1852 1853 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1854 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1855 : ResVReg; 1856 1857 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1858 .addDef(Sum) 1859 .addUse(GR.getSPIRVTypeID(ResType)) 1860 .addUse(Res) 1861 .addUse(Elt) 1862 .constrainAllUses(TII, TRI, RBI); 1863 Res = Sum; 1864 } 1865 1866 return Result; 1867 } 1868 1869 template <bool Signed> 1870 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1871 const SPIRVType *ResType, 1872 MachineInstr &I) const { 1873 assert(I.getNumOperands() == 5); 1874 assert(I.getOperand(2).isReg()); 1875 assert(I.getOperand(3).isReg()); 1876 assert(I.getOperand(4).isReg()); 1877 MachineBasicBlock &BB = *I.getParent(); 1878 1879 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1880 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1881 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1882 .addDef(Dot) 1883 .addUse(GR.getSPIRVTypeID(ResType)) 1884 .addUse(I.getOperand(2).getReg()) 1885 .addUse(I.getOperand(3).getReg()) 1886 .constrainAllUses(TII, TRI, RBI); 1887 1888 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1889 .addDef(ResVReg) 1890 .addUse(GR.getSPIRVTypeID(ResType)) 1891 .addUse(Dot) 1892 .addUse(I.getOperand(4).getReg()) 1893 .constrainAllUses(TII, TRI, RBI); 1894 } 1895 1896 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1897 // extract the elements of the packed inputs, multiply them and add the result 1898 // to the accumulator. 1899 template <bool Signed> 1900 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1901 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1902 assert(I.getNumOperands() == 5); 1903 assert(I.getOperand(2).isReg()); 1904 assert(I.getOperand(3).isReg()); 1905 assert(I.getOperand(4).isReg()); 1906 MachineBasicBlock &BB = *I.getParent(); 1907 1908 bool Result = true; 1909 1910 // Acc = C 1911 Register Acc = I.getOperand(4).getReg(); 1912 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1913 auto ExtractOp = 1914 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1915 1916 // Extract the i8 element, multiply and add it to the accumulator 1917 for (unsigned i = 0; i < 4; i++) { 1918 // A[i] 1919 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1920 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1921 .addDef(AElt) 1922 .addUse(GR.getSPIRVTypeID(ResType)) 1923 .addUse(I.getOperand(2).getReg()) 1924 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1925 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1926 .constrainAllUses(TII, TRI, RBI); 1927 1928 // B[i] 1929 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1930 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1931 .addDef(BElt) 1932 .addUse(GR.getSPIRVTypeID(ResType)) 1933 .addUse(I.getOperand(3).getReg()) 1934 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1935 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1936 .constrainAllUses(TII, TRI, RBI); 1937 1938 // A[i] * B[i] 1939 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1940 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1941 .addDef(Mul) 1942 .addUse(GR.getSPIRVTypeID(ResType)) 1943 .addUse(AElt) 1944 .addUse(BElt) 1945 .constrainAllUses(TII, TRI, RBI); 1946 1947 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1948 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1949 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1950 .addDef(MaskMul) 1951 .addUse(GR.getSPIRVTypeID(ResType)) 1952 .addUse(Mul) 1953 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1954 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1955 .constrainAllUses(TII, TRI, RBI); 1956 1957 // Acc = Acc + A[i] * B[i] 1958 Register Sum = 1959 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1960 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1961 .addDef(Sum) 1962 .addUse(GR.getSPIRVTypeID(ResType)) 1963 .addUse(Acc) 1964 .addUse(MaskMul) 1965 .constrainAllUses(TII, TRI, RBI); 1966 1967 Acc = Sum; 1968 } 1969 1970 return Result; 1971 } 1972 1973 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1974 /// does not have a saturate builtin. 1975 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1976 const SPIRVType *ResType, 1977 MachineInstr &I) const { 1978 assert(I.getNumOperands() == 3); 1979 assert(I.getOperand(2).isReg()); 1980 MachineBasicBlock &BB = *I.getParent(); 1981 Register VZero = buildZerosValF(ResType, I); 1982 Register VOne = buildOnesValF(ResType, I); 1983 1984 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1985 .addDef(ResVReg) 1986 .addUse(GR.getSPIRVTypeID(ResType)) 1987 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1988 .addImm(GL::FClamp) 1989 .addUse(I.getOperand(2).getReg()) 1990 .addUse(VZero) 1991 .addUse(VOne) 1992 .constrainAllUses(TII, TRI, RBI); 1993 } 1994 1995 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1996 const SPIRVType *ResType, 1997 MachineInstr &I) const { 1998 assert(I.getNumOperands() == 3); 1999 assert(I.getOperand(2).isReg()); 2000 MachineBasicBlock &BB = *I.getParent(); 2001 Register InputRegister = I.getOperand(2).getReg(); 2002 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 2003 auto &DL = I.getDebugLoc(); 2004 2005 if (!InputType) 2006 report_fatal_error("Input Type could not be determined."); 2007 2008 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 2009 2010 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 2011 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 2012 2013 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 2014 2015 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 2016 Register SignReg = NeedsConversion 2017 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 2018 : ResVReg; 2019 2020 bool Result = 2021 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 2022 .addDef(SignReg) 2023 .addUse(GR.getSPIRVTypeID(InputType)) 2024 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2025 .addImm(SignOpcode) 2026 .addUse(InputRegister) 2027 .constrainAllUses(TII, TRI, RBI); 2028 2029 if (NeedsConversion) { 2030 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2031 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2032 .addDef(ResVReg) 2033 .addUse(GR.getSPIRVTypeID(ResType)) 2034 .addUse(SignReg) 2035 .constrainAllUses(TII, TRI, RBI); 2036 } 2037 2038 return Result; 2039 } 2040 2041 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2042 const SPIRVType *ResType, 2043 MachineInstr &I, 2044 unsigned Opcode) const { 2045 MachineBasicBlock &BB = *I.getParent(); 2046 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2047 2048 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2049 .addDef(ResVReg) 2050 .addUse(GR.getSPIRVTypeID(ResType)) 2051 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2052 IntTy, TII)); 2053 2054 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2055 BMI.addUse(I.getOperand(J).getReg()); 2056 } 2057 2058 return BMI.constrainAllUses(TII, TRI, RBI); 2059 } 2060 2061 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2062 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2063 2064 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2065 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2066 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2067 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2068 SPIRV::OpGroupNonUniformBallot); 2069 2070 MachineBasicBlock &BB = *I.getParent(); 2071 Result &= 2072 BuildMI(BB, I, I.getDebugLoc(), 2073 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2074 .addDef(ResVReg) 2075 .addUse(GR.getSPIRVTypeID(ResType)) 2076 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2077 .addImm(SPIRV::GroupOperation::Reduce) 2078 .addUse(BallotReg) 2079 .constrainAllUses(TII, TRI, RBI); 2080 2081 return Result; 2082 } 2083 2084 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2085 const SPIRVType *ResType, 2086 MachineInstr &I) const { 2087 MachineBasicBlock &BB = *I.getParent(); 2088 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2089 .addDef(ResVReg) 2090 .addUse(GR.getSPIRVTypeID(ResType)) 2091 .addUse(I.getOperand(1).getReg()) 2092 .constrainAllUses(TII, TRI, RBI); 2093 } 2094 2095 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2096 const SPIRVType *ResType, 2097 MachineInstr &I) const { 2098 // There is no way to implement `freeze` correctly without support on SPIR-V 2099 // standard side, but we may at least address a simple (static) case when 2100 // undef/poison value presence is obvious. The main benefit of even 2101 // incomplete `freeze` support is preventing of translation from crashing due 2102 // to lack of support on legalization and instruction selection steps. 2103 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2104 return false; 2105 Register OpReg = I.getOperand(1).getReg(); 2106 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2107 Register Reg; 2108 switch (Def->getOpcode()) { 2109 case SPIRV::ASSIGN_TYPE: 2110 if (MachineInstr *AssignToDef = 2111 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2112 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2113 Reg = Def->getOperand(2).getReg(); 2114 } 2115 break; 2116 case SPIRV::OpUndef: 2117 Reg = Def->getOperand(1).getReg(); 2118 break; 2119 } 2120 unsigned DestOpCode; 2121 if (Reg.isValid()) { 2122 DestOpCode = SPIRV::OpConstantNull; 2123 } else { 2124 DestOpCode = TargetOpcode::COPY; 2125 Reg = OpReg; 2126 } 2127 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2128 .addDef(I.getOperand(0).getReg()) 2129 .addUse(Reg) 2130 .constrainAllUses(TII, TRI, RBI); 2131 } 2132 return false; 2133 } 2134 2135 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2136 const SPIRVType *ResType) { 2137 Register OpReg = ResType->getOperand(2).getReg(); 2138 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2139 if (!OpDef) 2140 return 0; 2141 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2142 OpDef->getOperand(1).isReg()) { 2143 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2144 OpDef = RefDef; 2145 } 2146 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2147 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2148 : 0; 2149 return N; 2150 } 2151 2152 // Return true if the type represents a constant register 2153 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2154 SmallPtrSet<SPIRVType *, 4> &Visited) { 2155 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2156 OpDef->getOperand(1).isReg()) { 2157 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2158 OpDef = RefDef; 2159 } 2160 2161 if (Visited.contains(OpDef)) 2162 return true; 2163 Visited.insert(OpDef); 2164 2165 unsigned Opcode = OpDef->getOpcode(); 2166 switch (Opcode) { 2167 case TargetOpcode::G_CONSTANT: 2168 case TargetOpcode::G_FCONSTANT: 2169 return true; 2170 case TargetOpcode::G_INTRINSIC: 2171 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2172 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2173 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2174 Intrinsic::spv_const_composite; 2175 case TargetOpcode::G_BUILD_VECTOR: 2176 case TargetOpcode::G_SPLAT_VECTOR: { 2177 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2178 i++) { 2179 SPIRVType *OpNestedDef = 2180 OpDef->getOperand(i).isReg() 2181 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2182 : nullptr; 2183 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2184 return false; 2185 } 2186 return true; 2187 } 2188 } 2189 return false; 2190 } 2191 2192 // Return true if the virtual register represents a constant 2193 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2194 SmallPtrSet<SPIRVType *, 4> Visited; 2195 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2196 return isConstReg(MRI, OpDef, Visited); 2197 return false; 2198 } 2199 2200 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2201 const SPIRVType *ResType, 2202 MachineInstr &I) const { 2203 unsigned N = 0; 2204 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2205 N = GR.getScalarOrVectorComponentCount(ResType); 2206 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2207 N = getArrayComponentCount(MRI, ResType); 2208 else 2209 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2210 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2211 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2212 2213 // check if we may construct a constant vector 2214 bool IsConst = true; 2215 for (unsigned i = I.getNumExplicitDefs(); 2216 i < I.getNumExplicitOperands() && IsConst; ++i) 2217 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2218 IsConst = false; 2219 2220 if (!IsConst && N < 2) 2221 report_fatal_error( 2222 "There must be at least two constituent operands in a vector"); 2223 2224 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2225 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2226 TII.get(IsConst ? SPIRV::OpConstantComposite 2227 : SPIRV::OpCompositeConstruct)) 2228 .addDef(ResVReg) 2229 .addUse(GR.getSPIRVTypeID(ResType)); 2230 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2231 MIB.addUse(I.getOperand(i).getReg()); 2232 return MIB.constrainAllUses(TII, TRI, RBI); 2233 } 2234 2235 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2236 const SPIRVType *ResType, 2237 MachineInstr &I) const { 2238 unsigned N = 0; 2239 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2240 N = GR.getScalarOrVectorComponentCount(ResType); 2241 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2242 N = getArrayComponentCount(MRI, ResType); 2243 else 2244 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2245 2246 unsigned OpIdx = I.getNumExplicitDefs(); 2247 if (!I.getOperand(OpIdx).isReg()) 2248 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2249 2250 // check if we may construct a constant vector 2251 Register OpReg = I.getOperand(OpIdx).getReg(); 2252 bool IsConst = isConstReg(MRI, OpReg); 2253 2254 if (!IsConst && N < 2) 2255 report_fatal_error( 2256 "There must be at least two constituent operands in a vector"); 2257 2258 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2259 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2260 TII.get(IsConst ? SPIRV::OpConstantComposite 2261 : SPIRV::OpCompositeConstruct)) 2262 .addDef(ResVReg) 2263 .addUse(GR.getSPIRVTypeID(ResType)); 2264 for (unsigned i = 0; i < N; ++i) 2265 MIB.addUse(OpReg); 2266 return MIB.constrainAllUses(TII, TRI, RBI); 2267 } 2268 2269 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2270 const SPIRVType *ResType, 2271 MachineInstr &I) const { 2272 2273 unsigned Opcode; 2274 2275 if (STI.canUseExtension( 2276 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2277 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2278 Opcode = SPIRV::OpDemoteToHelperInvocation; 2279 } else { 2280 Opcode = SPIRV::OpKill; 2281 // OpKill must be the last operation of any basic block. 2282 if (MachineInstr *NextI = I.getNextNode()) { 2283 GR.invalidateMachineInstr(NextI); 2284 NextI->removeFromParent(); 2285 } 2286 } 2287 2288 MachineBasicBlock &BB = *I.getParent(); 2289 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2290 .constrainAllUses(TII, TRI, RBI); 2291 } 2292 2293 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2294 const SPIRVType *ResType, 2295 unsigned CmpOpc, 2296 MachineInstr &I) const { 2297 Register Cmp0 = I.getOperand(2).getReg(); 2298 Register Cmp1 = I.getOperand(3).getReg(); 2299 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2300 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2301 "CMP operands should have the same type"); 2302 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2303 .addDef(ResVReg) 2304 .addUse(GR.getSPIRVTypeID(ResType)) 2305 .addUse(Cmp0) 2306 .addUse(Cmp1) 2307 .constrainAllUses(TII, TRI, RBI); 2308 } 2309 2310 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2311 const SPIRVType *ResType, 2312 MachineInstr &I) const { 2313 auto Pred = I.getOperand(1).getPredicate(); 2314 unsigned CmpOpc; 2315 2316 Register CmpOperand = I.getOperand(2).getReg(); 2317 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2318 CmpOpc = getPtrCmpOpcode(Pred); 2319 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2320 CmpOpc = getBoolCmpOpcode(Pred); 2321 else 2322 CmpOpc = getICmpOpcode(Pred); 2323 return selectCmp(ResVReg, ResType, CmpOpc, I); 2324 } 2325 2326 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2327 const MachineInstr &I, 2328 int OpIdx) const { 2329 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2330 "Expected G_FCONSTANT"); 2331 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2332 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2333 } 2334 2335 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2336 const MachineInstr &I, 2337 int OpIdx) const { 2338 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2339 "Expected G_CONSTANT"); 2340 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2341 } 2342 2343 std::pair<Register, bool> 2344 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2345 const SPIRVType *ResType) const { 2346 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2347 const SPIRVType *SpvI32Ty = 2348 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2349 // Find a constant in DT or build a new one. 2350 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2351 Register NewReg = GR.find(ConstInt, GR.CurMF); 2352 bool Result = true; 2353 if (!NewReg.isValid()) { 2354 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2355 GR.add(ConstInt, GR.CurMF, NewReg); 2356 MachineInstr *MI; 2357 MachineBasicBlock &BB = *I.getParent(); 2358 if (Val == 0) { 2359 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2360 .addDef(NewReg) 2361 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2362 } else { 2363 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2364 .addDef(NewReg) 2365 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2366 .addImm(APInt(32, Val).getZExtValue()); 2367 } 2368 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2369 } 2370 return {NewReg, Result}; 2371 } 2372 2373 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2374 const SPIRVType *ResType, 2375 MachineInstr &I) const { 2376 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2377 return selectCmp(ResVReg, ResType, CmpOp, I); 2378 } 2379 2380 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2381 MachineInstr &I) const { 2382 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2383 bool ZeroAsNull = STI.isOpenCLEnv(); 2384 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2385 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2386 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2387 } 2388 2389 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2390 MachineInstr &I) const { 2391 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2392 bool ZeroAsNull = STI.isOpenCLEnv(); 2393 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2394 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2395 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2396 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2397 } 2398 2399 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2400 MachineInstr &I) const { 2401 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2402 bool ZeroAsNull = STI.isOpenCLEnv(); 2403 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2404 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2405 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2406 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2407 } 2408 2409 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2410 const SPIRVType *ResType, 2411 MachineInstr &I) const { 2412 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2413 APInt One = 2414 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2415 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2416 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2417 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2418 } 2419 2420 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2421 const SPIRVType *ResType, 2422 MachineInstr &I, 2423 bool IsSigned) const { 2424 // To extend a bool, we need to use OpSelect between constants. 2425 Register ZeroReg = buildZerosVal(ResType, I); 2426 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2427 bool IsScalarBool = 2428 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2429 unsigned Opcode = 2430 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2431 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2432 .addDef(ResVReg) 2433 .addUse(GR.getSPIRVTypeID(ResType)) 2434 .addUse(I.getOperand(1).getReg()) 2435 .addUse(OneReg) 2436 .addUse(ZeroReg) 2437 .constrainAllUses(TII, TRI, RBI); 2438 } 2439 2440 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2441 const SPIRVType *ResType, 2442 MachineInstr &I, bool IsSigned, 2443 unsigned Opcode) const { 2444 Register SrcReg = I.getOperand(1).getReg(); 2445 // We can convert bool value directly to float type without OpConvert*ToF, 2446 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2447 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2448 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2449 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2450 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2451 const unsigned NumElts = ResType->getOperand(2).getImm(); 2452 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2453 } 2454 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2455 selectSelect(SrcReg, TmpType, I, false); 2456 } 2457 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2458 } 2459 2460 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2461 const SPIRVType *ResType, 2462 MachineInstr &I, bool IsSigned) const { 2463 Register SrcReg = I.getOperand(1).getReg(); 2464 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2465 return selectSelect(ResVReg, ResType, I, IsSigned); 2466 2467 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2468 if (SrcType == ResType) 2469 return BuildCOPY(ResVReg, SrcReg, I); 2470 2471 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2472 return selectUnOp(ResVReg, ResType, I, Opcode); 2473 } 2474 2475 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2476 const SPIRVType *ResType, 2477 MachineInstr &I, 2478 bool IsSigned) const { 2479 MachineIRBuilder MIRBuilder(I); 2480 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2481 MachineBasicBlock &BB = *I.getParent(); 2482 // Ensure we have bool. 2483 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2484 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2485 if (N > 1) 2486 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2487 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2488 // Build less-than-equal and less-than. 2489 // TODO: replace with one-liner createVirtualRegister() from 2490 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2491 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2492 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2493 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2494 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2495 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2496 : SPIRV::OpULessThanEqual)) 2497 .addDef(IsLessEqReg) 2498 .addUse(BoolTypeReg) 2499 .addUse(I.getOperand(1).getReg()) 2500 .addUse(I.getOperand(2).getReg()) 2501 .constrainAllUses(TII, TRI, RBI); 2502 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2503 MRI->setType(IsLessReg, LLT::scalar(64)); 2504 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2505 Result &= BuildMI(BB, I, I.getDebugLoc(), 2506 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2507 .addDef(IsLessReg) 2508 .addUse(BoolTypeReg) 2509 .addUse(I.getOperand(1).getReg()) 2510 .addUse(I.getOperand(2).getReg()) 2511 .constrainAllUses(TII, TRI, RBI); 2512 // Build selects. 2513 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2514 Register NegOneOrZeroReg = 2515 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2516 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2517 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2518 unsigned SelectOpcode = 2519 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2520 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2521 .addDef(NegOneOrZeroReg) 2522 .addUse(ResTypeReg) 2523 .addUse(IsLessReg) 2524 .addUse(buildOnesVal(true, ResType, I)) // -1 2525 .addUse(buildZerosVal(ResType, I)) 2526 .constrainAllUses(TII, TRI, RBI); 2527 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2528 .addDef(ResVReg) 2529 .addUse(ResTypeReg) 2530 .addUse(IsLessEqReg) 2531 .addUse(NegOneOrZeroReg) // -1 or 0 2532 .addUse(buildOnesVal(false, ResType, I)) 2533 .constrainAllUses(TII, TRI, RBI); 2534 } 2535 2536 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2537 Register ResVReg, 2538 MachineInstr &I, 2539 const SPIRVType *IntTy, 2540 const SPIRVType *BoolTy) const { 2541 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2542 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2543 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2544 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2545 Register Zero = buildZerosVal(IntTy, I); 2546 Register One = buildOnesVal(false, IntTy, I); 2547 MachineBasicBlock &BB = *I.getParent(); 2548 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2549 .addDef(BitIntReg) 2550 .addUse(GR.getSPIRVTypeID(IntTy)) 2551 .addUse(IntReg) 2552 .addUse(One) 2553 .constrainAllUses(TII, TRI, RBI); 2554 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2555 .addDef(ResVReg) 2556 .addUse(GR.getSPIRVTypeID(BoolTy)) 2557 .addUse(BitIntReg) 2558 .addUse(Zero) 2559 .constrainAllUses(TII, TRI, RBI); 2560 } 2561 2562 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2563 const SPIRVType *ResType, 2564 MachineInstr &I) const { 2565 Register IntReg = I.getOperand(1).getReg(); 2566 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2567 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2568 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2569 if (ArgType == ResType) 2570 return BuildCOPY(ResVReg, IntReg, I); 2571 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2572 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2573 return selectUnOp(ResVReg, ResType, I, Opcode); 2574 } 2575 2576 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2577 const SPIRVType *ResType, 2578 const APInt &Imm, 2579 MachineInstr &I) const { 2580 unsigned TyOpcode = ResType->getOpcode(); 2581 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2582 MachineBasicBlock &BB = *I.getParent(); 2583 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2584 Imm.isZero()) 2585 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2586 .addDef(ResVReg) 2587 .addUse(GR.getSPIRVTypeID(ResType)) 2588 .constrainAllUses(TII, TRI, RBI); 2589 if (TyOpcode == SPIRV::OpTypeInt) { 2590 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2591 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2592 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2593 } 2594 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2595 .addDef(ResVReg) 2596 .addUse(GR.getSPIRVTypeID(ResType)); 2597 // <=32-bit integers should be caught by the sdag pattern. 2598 assert(Imm.getBitWidth() > 32); 2599 addNumImm(Imm, MIB); 2600 return MIB.constrainAllUses(TII, TRI, RBI); 2601 } 2602 2603 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2604 const SPIRVType *ResType, 2605 MachineInstr &I) const { 2606 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2607 .addDef(ResVReg) 2608 .addUse(GR.getSPIRVTypeID(ResType)) 2609 .constrainAllUses(TII, TRI, RBI); 2610 } 2611 2612 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2613 assert(MO.isReg()); 2614 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2615 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2616 assert(TypeInst->getOperand(1).isReg()); 2617 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2618 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2619 } 2620 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2621 } 2622 2623 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2624 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2625 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2626 return TypeInst->getOperand(2).getImm(); 2627 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2628 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2629 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2630 } 2631 2632 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2633 const SPIRVType *ResType, 2634 MachineInstr &I) const { 2635 MachineBasicBlock &BB = *I.getParent(); 2636 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2637 .addDef(ResVReg) 2638 .addUse(GR.getSPIRVTypeID(ResType)) 2639 // object to insert 2640 .addUse(I.getOperand(3).getReg()) 2641 // composite to insert into 2642 .addUse(I.getOperand(2).getReg()); 2643 for (unsigned i = 4; i < I.getNumOperands(); i++) 2644 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2645 return MIB.constrainAllUses(TII, TRI, RBI); 2646 } 2647 2648 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2649 const SPIRVType *ResType, 2650 MachineInstr &I) const { 2651 MachineBasicBlock &BB = *I.getParent(); 2652 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2653 .addDef(ResVReg) 2654 .addUse(GR.getSPIRVTypeID(ResType)) 2655 .addUse(I.getOperand(2).getReg()); 2656 for (unsigned i = 3; i < I.getNumOperands(); i++) 2657 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2658 return MIB.constrainAllUses(TII, TRI, RBI); 2659 } 2660 2661 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2662 const SPIRVType *ResType, 2663 MachineInstr &I) const { 2664 if (isImm(I.getOperand(4), MRI)) 2665 return selectInsertVal(ResVReg, ResType, I); 2666 MachineBasicBlock &BB = *I.getParent(); 2667 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2668 .addDef(ResVReg) 2669 .addUse(GR.getSPIRVTypeID(ResType)) 2670 .addUse(I.getOperand(2).getReg()) 2671 .addUse(I.getOperand(3).getReg()) 2672 .addUse(I.getOperand(4).getReg()) 2673 .constrainAllUses(TII, TRI, RBI); 2674 } 2675 2676 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2677 const SPIRVType *ResType, 2678 MachineInstr &I) const { 2679 if (isImm(I.getOperand(3), MRI)) 2680 return selectExtractVal(ResVReg, ResType, I); 2681 MachineBasicBlock &BB = *I.getParent(); 2682 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2683 .addDef(ResVReg) 2684 .addUse(GR.getSPIRVTypeID(ResType)) 2685 .addUse(I.getOperand(2).getReg()) 2686 .addUse(I.getOperand(3).getReg()) 2687 .constrainAllUses(TII, TRI, RBI); 2688 } 2689 2690 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2691 const SPIRVType *ResType, 2692 MachineInstr &I) const { 2693 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2694 2695 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2696 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2697 // we have to use Op[InBounds]AccessChain. 2698 const unsigned Opcode = STI.isVulkanEnv() 2699 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2700 : SPIRV::OpAccessChain) 2701 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2702 : SPIRV::OpPtrAccessChain); 2703 2704 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2705 .addDef(ResVReg) 2706 .addUse(GR.getSPIRVTypeID(ResType)) 2707 // Object to get a pointer to. 2708 .addUse(I.getOperand(3).getReg()); 2709 // Adding indices. 2710 const unsigned StartingIndex = 2711 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2712 ? 5 2713 : 4; 2714 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2715 Res.addUse(I.getOperand(i).getReg()); 2716 return Res.constrainAllUses(TII, TRI, RBI); 2717 } 2718 2719 // Maybe wrap a value into OpSpecConstantOp 2720 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2721 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2722 bool Result = true; 2723 unsigned Lim = I.getNumExplicitOperands(); 2724 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2725 Register OpReg = I.getOperand(i).getReg(); 2726 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2727 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2728 SmallPtrSet<SPIRVType *, 4> Visited; 2729 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2730 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2731 GR.isAggregateType(OpType)) { 2732 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2733 // by selectAddrSpaceCast() 2734 CompositeArgs.push_back(OpReg); 2735 continue; 2736 } 2737 MachineFunction *MF = I.getMF(); 2738 Register WrapReg = GR.find(OpDefine, MF); 2739 if (WrapReg.isValid()) { 2740 CompositeArgs.push_back(WrapReg); 2741 continue; 2742 } 2743 // Create a new register for the wrapper 2744 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2745 GR.add(OpDefine, MF, WrapReg); 2746 CompositeArgs.push_back(WrapReg); 2747 // Decorate the wrapper register and generate a new instruction 2748 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2749 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2750 MachineBasicBlock &BB = *I.getParent(); 2751 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2752 .addDef(WrapReg) 2753 .addUse(GR.getSPIRVTypeID(OpType)) 2754 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2755 .addUse(OpReg) 2756 .constrainAllUses(TII, TRI, RBI); 2757 if (!Result) 2758 break; 2759 } 2760 return Result; 2761 } 2762 2763 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2764 const SPIRVType *ResType, 2765 MachineInstr &I) const { 2766 MachineBasicBlock &BB = *I.getParent(); 2767 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2768 switch (IID) { 2769 case Intrinsic::spv_load: 2770 return selectLoad(ResVReg, ResType, I); 2771 case Intrinsic::spv_store: 2772 return selectStore(I); 2773 case Intrinsic::spv_extractv: 2774 return selectExtractVal(ResVReg, ResType, I); 2775 case Intrinsic::spv_insertv: 2776 return selectInsertVal(ResVReg, ResType, I); 2777 case Intrinsic::spv_extractelt: 2778 return selectExtractElt(ResVReg, ResType, I); 2779 case Intrinsic::spv_insertelt: 2780 return selectInsertElt(ResVReg, ResType, I); 2781 case Intrinsic::spv_gep: 2782 return selectGEP(ResVReg, ResType, I); 2783 case Intrinsic::spv_unref_global: 2784 case Intrinsic::spv_init_global: { 2785 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2786 MachineInstr *Init = I.getNumExplicitOperands() > 2 2787 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2788 : nullptr; 2789 assert(MI); 2790 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2791 } 2792 case Intrinsic::spv_undef: { 2793 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2794 .addDef(ResVReg) 2795 .addUse(GR.getSPIRVTypeID(ResType)); 2796 return MIB.constrainAllUses(TII, TRI, RBI); 2797 } 2798 case Intrinsic::spv_const_composite: { 2799 // If no values are attached, the composite is null constant. 2800 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2801 // Select a proper instruction. 2802 unsigned Opcode = SPIRV::OpConstantNull; 2803 SmallVector<Register> CompositeArgs; 2804 if (!IsNull) { 2805 Opcode = SPIRV::OpConstantComposite; 2806 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2807 return false; 2808 } 2809 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2810 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2811 .addDef(ResVReg) 2812 .addUse(GR.getSPIRVTypeID(ResType)); 2813 // skip type MD node we already used when generated assign.type for this 2814 if (!IsNull) { 2815 for (Register OpReg : CompositeArgs) 2816 MIB.addUse(OpReg); 2817 } 2818 return MIB.constrainAllUses(TII, TRI, RBI); 2819 } 2820 case Intrinsic::spv_assign_name: { 2821 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2822 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2823 for (unsigned i = I.getNumExplicitDefs() + 2; 2824 i < I.getNumExplicitOperands(); ++i) { 2825 MIB.addImm(I.getOperand(i).getImm()); 2826 } 2827 return MIB.constrainAllUses(TII, TRI, RBI); 2828 } 2829 case Intrinsic::spv_switch: { 2830 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2831 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2832 if (I.getOperand(i).isReg()) 2833 MIB.addReg(I.getOperand(i).getReg()); 2834 else if (I.getOperand(i).isCImm()) 2835 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2836 else if (I.getOperand(i).isMBB()) 2837 MIB.addMBB(I.getOperand(i).getMBB()); 2838 else 2839 llvm_unreachable("Unexpected OpSwitch operand"); 2840 } 2841 return MIB.constrainAllUses(TII, TRI, RBI); 2842 } 2843 case Intrinsic::spv_loop_merge: { 2844 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge)); 2845 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2846 assert(I.getOperand(i).isMBB()); 2847 MIB.addMBB(I.getOperand(i).getMBB()); 2848 } 2849 MIB.addImm(SPIRV::SelectionControl::None); 2850 return MIB.constrainAllUses(TII, TRI, RBI); 2851 } 2852 case Intrinsic::spv_selection_merge: { 2853 auto MIB = 2854 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge)); 2855 assert(I.getOperand(1).isMBB() && 2856 "operand 1 to spv_selection_merge must be a basic block"); 2857 MIB.addMBB(I.getOperand(1).getMBB()); 2858 MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm())); 2859 return MIB.constrainAllUses(TII, TRI, RBI); 2860 } 2861 case Intrinsic::spv_cmpxchg: 2862 return selectAtomicCmpXchg(ResVReg, ResType, I); 2863 case Intrinsic::spv_unreachable: 2864 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2865 .constrainAllUses(TII, TRI, RBI); 2866 case Intrinsic::spv_alloca: 2867 return selectFrameIndex(ResVReg, ResType, I); 2868 case Intrinsic::spv_alloca_array: 2869 return selectAllocaArray(ResVReg, ResType, I); 2870 case Intrinsic::spv_assume: 2871 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2872 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2873 .addUse(I.getOperand(1).getReg()) 2874 .constrainAllUses(TII, TRI, RBI); 2875 break; 2876 case Intrinsic::spv_expect: 2877 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2878 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2879 .addDef(ResVReg) 2880 .addUse(GR.getSPIRVTypeID(ResType)) 2881 .addUse(I.getOperand(2).getReg()) 2882 .addUse(I.getOperand(3).getReg()) 2883 .constrainAllUses(TII, TRI, RBI); 2884 break; 2885 case Intrinsic::arithmetic_fence: 2886 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2887 return BuildMI(BB, I, I.getDebugLoc(), 2888 TII.get(SPIRV::OpArithmeticFenceEXT)) 2889 .addDef(ResVReg) 2890 .addUse(GR.getSPIRVTypeID(ResType)) 2891 .addUse(I.getOperand(2).getReg()) 2892 .constrainAllUses(TII, TRI, RBI); 2893 else 2894 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2895 break; 2896 case Intrinsic::spv_thread_id: 2897 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 2898 // intrinsic in LLVM IR for SPIR-V backend. 2899 // 2900 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 2901 // `GlobalInvocationId` builtin variable 2902 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 2903 ResType, I); 2904 case Intrinsic::spv_thread_id_in_group: 2905 // The HLSL SV_GroupThreadId semantic is lowered to 2906 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 2907 // 2908 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 2909 // translated to a `LocalInvocationId` builtin variable 2910 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 2911 ResType, I); 2912 case Intrinsic::spv_group_id: 2913 // The HLSL SV_GroupId semantic is lowered to 2914 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. 2915 // 2916 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` 2917 // builtin variable 2918 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, 2919 I); 2920 case Intrinsic::spv_fdot: 2921 return selectFloatDot(ResVReg, ResType, I); 2922 case Intrinsic::spv_udot: 2923 case Intrinsic::spv_sdot: 2924 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2925 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2926 return selectIntegerDot(ResVReg, ResType, I, 2927 /*Signed=*/IID == Intrinsic::spv_sdot); 2928 return selectIntegerDotExpansion(ResVReg, ResType, I); 2929 case Intrinsic::spv_dot4add_i8packed: 2930 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2931 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2932 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2933 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2934 case Intrinsic::spv_dot4add_u8packed: 2935 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2936 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2937 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2938 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2939 case Intrinsic::spv_all: 2940 return selectAll(ResVReg, ResType, I); 2941 case Intrinsic::spv_any: 2942 return selectAny(ResVReg, ResType, I); 2943 case Intrinsic::spv_cross: 2944 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2945 case Intrinsic::spv_distance: 2946 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); 2947 case Intrinsic::spv_lerp: 2948 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2949 case Intrinsic::spv_length: 2950 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2951 case Intrinsic::spv_degrees: 2952 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2953 case Intrinsic::spv_frac: 2954 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2955 case Intrinsic::spv_normalize: 2956 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2957 case Intrinsic::spv_rsqrt: 2958 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2959 case Intrinsic::spv_sign: 2960 return selectSign(ResVReg, ResType, I); 2961 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2962 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2963 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2964 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2965 case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb 2966 return selectFirstBitLow(ResVReg, ResType, I); 2967 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2968 bool Result = true; 2969 auto MemSemConstant = 2970 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2971 Register MemSemReg = MemSemConstant.first; 2972 Result &= MemSemConstant.second; 2973 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2974 Register ScopeReg = ScopeConstant.first; 2975 Result &= ScopeConstant.second; 2976 MachineBasicBlock &BB = *I.getParent(); 2977 return Result && 2978 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2979 .addUse(ScopeReg) 2980 .addUse(ScopeReg) 2981 .addUse(MemSemReg) 2982 .constrainAllUses(TII, TRI, RBI); 2983 } 2984 case Intrinsic::spv_lifetime_start: 2985 case Intrinsic::spv_lifetime_end: { 2986 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2987 : SPIRV::OpLifetimeStop; 2988 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2989 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2990 if (Size == -1) 2991 Size = 0; 2992 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2993 .addUse(PtrReg) 2994 .addImm(Size) 2995 .constrainAllUses(TII, TRI, RBI); 2996 } 2997 case Intrinsic::spv_saturate: 2998 return selectSaturate(ResVReg, ResType, I); 2999 case Intrinsic::spv_nclamp: 3000 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 3001 case Intrinsic::spv_uclamp: 3002 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 3003 case Intrinsic::spv_sclamp: 3004 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 3005 case Intrinsic::spv_wave_active_countbits: 3006 return selectWaveActiveCountBits(ResVReg, ResType, I); 3007 case Intrinsic::spv_wave_all: 3008 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 3009 case Intrinsic::spv_wave_any: 3010 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 3011 case Intrinsic::spv_wave_is_first_lane: 3012 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 3013 case Intrinsic::spv_wave_readlane: 3014 return selectWaveOpInst(ResVReg, ResType, I, 3015 SPIRV::OpGroupNonUniformShuffle); 3016 case Intrinsic::spv_step: 3017 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 3018 case Intrinsic::spv_radians: 3019 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 3020 // Discard intrinsics which we do not expect to actually represent code after 3021 // lowering or intrinsics which are not implemented but should not crash when 3022 // found in a customer's LLVM IR input. 3023 case Intrinsic::instrprof_increment: 3024 case Intrinsic::instrprof_increment_step: 3025 case Intrinsic::instrprof_value_profile: 3026 break; 3027 // Discard internal intrinsics. 3028 case Intrinsic::spv_value_md: 3029 break; 3030 case Intrinsic::spv_resource_handlefrombinding: { 3031 return selectHandleFromBinding(ResVReg, ResType, I); 3032 } 3033 case Intrinsic::spv_resource_store_typedbuffer: { 3034 return selectImageWriteIntrinsic(I); 3035 } 3036 case Intrinsic::spv_resource_load_typedbuffer: { 3037 return selectReadImageIntrinsic(ResVReg, ResType, I); 3038 } 3039 case Intrinsic::spv_discard: { 3040 return selectDiscard(ResVReg, ResType, I); 3041 } 3042 default: { 3043 std::string DiagMsg; 3044 raw_string_ostream OS(DiagMsg); 3045 I.print(OS); 3046 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3047 report_fatal_error(DiagMsg.c_str(), false); 3048 } 3049 } 3050 return true; 3051 } 3052 3053 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3054 const SPIRVType *ResType, 3055 MachineInstr &I) const { 3056 3057 uint32_t Set = foldImm(I.getOperand(2), MRI); 3058 uint32_t Binding = foldImm(I.getOperand(3), MRI); 3059 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 3060 Register IndexReg = I.getOperand(5).getReg(); 3061 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 3062 3063 MachineIRBuilder MIRBuilder(I); 3064 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3065 IndexReg, IsNonUniform, MIRBuilder); 3066 3067 if (IsNonUniform) 3068 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 3069 3070 // TODO: For now we assume the resource is an image, which needs to be 3071 // loaded to get the handle. That will not be true for storage buffers. 3072 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3073 .addDef(ResVReg) 3074 .addUse(GR.getSPIRVTypeID(ResType)) 3075 .addUse(VarReg) 3076 .constrainAllUses(TII, TRI, RBI); 3077 } 3078 3079 bool SPIRVInstructionSelector::selectReadImageIntrinsic( 3080 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3081 3082 // If the load of the image is in a different basic block, then 3083 // this will generate invalid code. A proper solution is to move 3084 // the OpLoad from selectHandleFromBinding here. However, to do 3085 // that we will need to change the return type of the intrinsic. 3086 // We will do that when we can, but for now trying to move forward with other 3087 // issues. 3088 Register ImageReg = I.getOperand(2).getReg(); 3089 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3090 "The image must be loaded in the same basic block as its use."); 3091 3092 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3093 if (ResultSize == 4) { 3094 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3095 TII.get(SPIRV::OpImageRead)) 3096 .addDef(ResVReg) 3097 .addUse(GR.getSPIRVTypeID(ResType)) 3098 .addUse(ImageReg) 3099 .addUse(I.getOperand(3).getReg()) 3100 .constrainAllUses(TII, TRI, RBI); 3101 } 3102 3103 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 3104 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3105 bool Succeed = 3106 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3107 .addDef(ReadReg) 3108 .addUse(GR.getSPIRVTypeID(ReadType)) 3109 .addUse(ImageReg) 3110 .addUse(I.getOperand(3).getReg()) 3111 .constrainAllUses(TII, TRI, RBI); 3112 if (!Succeed) 3113 return false; 3114 3115 if (ResultSize == 1) { 3116 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3117 TII.get(SPIRV::OpCompositeExtract)) 3118 .addDef(ResVReg) 3119 .addUse(GR.getSPIRVTypeID(ResType)) 3120 .addUse(ReadReg) 3121 .addImm(0) 3122 .constrainAllUses(TII, TRI, RBI); 3123 } 3124 return extractSubvector(ResVReg, ResType, ReadReg, I); 3125 } 3126 3127 bool SPIRVInstructionSelector::extractSubvector( 3128 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3129 MachineInstr &InsertionPoint) const { 3130 SPIRVType *InputType = GR.getResultType(ReadReg); 3131 [[maybe_unused]] uint64_t InputSize = 3132 GR.getScalarOrVectorComponentCount(InputType); 3133 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3134 assert(InputSize > 1 && "The input must be a vector."); 3135 assert(ResultSize > 1 && "The result must be a vector."); 3136 assert(ResultSize < InputSize && 3137 "Cannot extract more element than there are in the input."); 3138 SmallVector<Register> ComponentRegisters; 3139 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3140 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3141 for (uint64_t I = 0; I < ResultSize; I++) { 3142 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3143 bool Succeed = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3144 InsertionPoint.getDebugLoc(), 3145 TII.get(SPIRV::OpCompositeExtract)) 3146 .addDef(ComponentReg) 3147 .addUse(ScalarType->getOperand(0).getReg()) 3148 .addUse(ReadReg) 3149 .addImm(I) 3150 .constrainAllUses(TII, TRI, RBI); 3151 if (!Succeed) 3152 return false; 3153 ComponentRegisters.emplace_back(ComponentReg); 3154 } 3155 3156 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3157 InsertionPoint.getDebugLoc(), 3158 TII.get(SPIRV::OpCompositeConstruct)) 3159 .addDef(ResVReg) 3160 .addUse(GR.getSPIRVTypeID(ResType)); 3161 3162 for (Register ComponentReg : ComponentRegisters) 3163 MIB.addUse(ComponentReg); 3164 return MIB.constrainAllUses(TII, TRI, RBI); 3165 } 3166 3167 bool SPIRVInstructionSelector::selectImageWriteIntrinsic( 3168 MachineInstr &I) const { 3169 // If the load of the image is in a different basic block, then 3170 // this will generate invalid code. A proper solution is to move 3171 // the OpLoad from selectHandleFromBinding here. However, to do 3172 // that we will need to change the return type of the intrinsic. 3173 // We will do that when we can, but for now trying to move forward with other 3174 // issues. 3175 Register ImageReg = I.getOperand(1).getReg(); 3176 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3177 "The image must be loaded in the same basic block as its use."); 3178 Register CoordinateReg = I.getOperand(2).getReg(); 3179 Register DataReg = I.getOperand(3).getReg(); 3180 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3181 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3182 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 3183 TII.get(SPIRV::OpImageWrite)) 3184 .addUse(ImageReg) 3185 .addUse(CoordinateReg) 3186 .addUse(DataReg) 3187 .constrainAllUses(TII, TRI, RBI); 3188 } 3189 3190 Register SPIRVInstructionSelector::buildPointerToResource( 3191 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3192 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3193 MachineIRBuilder MIRBuilder) const { 3194 if (ArraySize == 1) 3195 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3196 MIRBuilder); 3197 3198 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3199 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3200 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3201 VarType, Set, Binding, MIRBuilder); 3202 3203 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3204 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3205 3206 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3207 if (IsNonUniform) { 3208 // It is unclear which value needs to be marked an non-uniform, so both 3209 // the index and the access changed are decorated as non-uniform. 3210 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3211 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3212 } 3213 3214 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3215 .addDef(AcReg) 3216 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3217 .addUse(VarReg) 3218 .addUse(IndexReg); 3219 3220 return AcReg; 3221 } 3222 3223 bool SPIRVInstructionSelector::selectFirstBitSet16( 3224 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3225 unsigned ExtendOpcode, unsigned BitSetOpcode) const { 3226 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3227 bool Result = selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, 3228 ExtendOpcode); 3229 3230 return Result && 3231 selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode); 3232 } 3233 3234 bool SPIRVInstructionSelector::selectFirstBitSet32( 3235 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3236 Register SrcReg, unsigned BitSetOpcode) const { 3237 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3238 .addDef(ResVReg) 3239 .addUse(GR.getSPIRVTypeID(ResType)) 3240 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3241 .addImm(BitSetOpcode) 3242 .addUse(SrcReg) 3243 .constrainAllUses(TII, TRI, RBI); 3244 } 3245 3246 bool SPIRVInstructionSelector::selectFirstBitSet64Overflow( 3247 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3248 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3249 3250 // SPIR-V allow vectors of size 2,3,4 only. Calling with a larger vectors 3251 // requires creating a param register and return register with an invalid 3252 // vector size. If that is resolved, then this function can be used for 3253 // vectors of any component size. 3254 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3255 assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops"); 3256 3257 MachineIRBuilder MIRBuilder(I); 3258 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3259 SPIRVType *I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder); 3260 SPIRVType *I64x2Type = GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder); 3261 SPIRVType *Vec2ResType = 3262 GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder); 3263 3264 std::vector<Register> PartialRegs; 3265 3266 // Loops 0, 2, 4, ... but stops one loop early when ComponentCount is odd 3267 unsigned CurrentComponent = 0; 3268 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) { 3269 // This register holds the firstbitX result for each of the i64x2 vectors 3270 // extracted from SrcReg 3271 Register BitSetResult = 3272 MRI->createVirtualRegister(GR.getRegClass(I64x2Type)); 3273 3274 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3275 TII.get(SPIRV::OpVectorShuffle)) 3276 .addDef(BitSetResult) 3277 .addUse(GR.getSPIRVTypeID(I64x2Type)) 3278 .addUse(SrcReg) 3279 .addUse(SrcReg) 3280 .addImm(CurrentComponent) 3281 .addImm(CurrentComponent + 1); 3282 3283 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3284 return false; 3285 3286 Register SubVecBitSetReg = 3287 MRI->createVirtualRegister(GR.getRegClass(Vec2ResType)); 3288 3289 if (!selectFirstBitSet64(SubVecBitSetReg, Vec2ResType, I, BitSetResult, 3290 BitSetOpcode, SwapPrimarySide)) 3291 return false; 3292 3293 PartialRegs.push_back(SubVecBitSetReg); 3294 } 3295 3296 // On odd component counts we need to handle one more component 3297 if (CurrentComponent != ComponentCount) { 3298 bool ZeroAsNull = STI.isOpenCLEnv(); 3299 Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type)); 3300 Register ConstIntLastIdx = GR.getOrCreateConstInt( 3301 ComponentCount - 1, I, BaseType, TII, ZeroAsNull); 3302 3303 if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx}, 3304 SPIRV::OpVectorExtractDynamic)) 3305 return false; 3306 3307 Register FinalElemBitSetReg = 3308 MRI->createVirtualRegister(GR.getRegClass(BaseType)); 3309 3310 if (!selectFirstBitSet64(FinalElemBitSetReg, BaseType, I, FinalElemReg, 3311 BitSetOpcode, SwapPrimarySide)) 3312 return false; 3313 3314 PartialRegs.push_back(FinalElemBitSetReg); 3315 } 3316 3317 // Join all the resulting registers back into the return type in order 3318 // (ie i32x2, i32x2, i32x1 -> i32x5) 3319 return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs, 3320 SPIRV::OpCompositeConstruct); 3321 } 3322 3323 bool SPIRVInstructionSelector::selectFirstBitSet64( 3324 Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 3325 Register SrcReg, unsigned BitSetOpcode, bool SwapPrimarySide) const { 3326 unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); 3327 SPIRVType *BaseType = GR.retrieveScalarOrVectorIntType(ResType); 3328 bool ZeroAsNull = STI.isOpenCLEnv(); 3329 Register ConstIntZero = 3330 GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull); 3331 Register ConstIntOne = 3332 GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull); 3333 3334 // SPIRV doesn't support vectors with more than 4 components. Since the 3335 // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only 3336 // operate on vectors with 2 or less components. When largers vectors are 3337 // seen. Split them, recurse, then recombine them. 3338 if (ComponentCount > 2) { 3339 return selectFirstBitSet64Overflow(ResVReg, ResType, I, SrcReg, 3340 BitSetOpcode, SwapPrimarySide); 3341 } 3342 3343 // 1. Split int64 into 2 pieces using a bitcast 3344 MachineIRBuilder MIRBuilder(I); 3345 SPIRVType *PostCastType = 3346 GR.getOrCreateSPIRVVectorType(BaseType, 2 * ComponentCount, MIRBuilder); 3347 Register BitcastReg = 3348 MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3349 3350 if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg}, 3351 SPIRV::OpBitcast)) 3352 return false; 3353 3354 // 2. Find the first set bit from the primary side for all the pieces in #1 3355 Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType)); 3356 if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode)) 3357 return false; 3358 3359 // 3. Split result vector into high bits and low bits 3360 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3361 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3362 3363 bool IsScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3364 if (IsScalarRes) { 3365 // if scalar do a vector extract 3366 if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntZero}, 3367 SPIRV::OpVectorExtractDynamic)) 3368 return false; 3369 if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntOne}, 3370 SPIRV::OpVectorExtractDynamic)) 3371 return false; 3372 } else { 3373 // if vector do a shufflevector 3374 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3375 TII.get(SPIRV::OpVectorShuffle)) 3376 .addDef(HighReg) 3377 .addUse(GR.getSPIRVTypeID(ResType)) 3378 .addUse(FBSReg) 3379 // Per the spec, repeat the vector if only one vec is needed 3380 .addUse(FBSReg); 3381 3382 // high bits are stored in even indexes. Extract them from FBSReg 3383 for (unsigned J = 0; J < ComponentCount * 2; J += 2) { 3384 MIB.addImm(J); 3385 } 3386 3387 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3388 return false; 3389 3390 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3391 TII.get(SPIRV::OpVectorShuffle)) 3392 .addDef(LowReg) 3393 .addUse(GR.getSPIRVTypeID(ResType)) 3394 .addUse(FBSReg) 3395 // Per the spec, repeat the vector if only one vec is needed 3396 .addUse(FBSReg); 3397 3398 // low bits are stored in odd indexes. Extract them from FBSReg 3399 for (unsigned J = 1; J < ComponentCount * 2; J += 2) { 3400 MIB.addImm(J); 3401 } 3402 if (!MIB.constrainAllUses(TII, TRI, RBI)) 3403 return false; 3404 } 3405 3406 // 4. Check the result. When primary bits == -1 use secondary, otherwise use 3407 // primary 3408 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3409 Register NegOneReg; 3410 Register Reg0; 3411 Register Reg32; 3412 unsigned SelectOp; 3413 unsigned AddOp; 3414 3415 if (IsScalarRes) { 3416 NegOneReg = 3417 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3418 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3419 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3420 SelectOp = SPIRV::OpSelectSISCond; 3421 AddOp = SPIRV::OpIAddS; 3422 } else { 3423 BoolType = 3424 GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount, MIRBuilder); 3425 NegOneReg = 3426 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3427 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3428 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3429 SelectOp = SPIRV::OpSelectVIVCond; 3430 AddOp = SPIRV::OpIAddV; 3431 } 3432 3433 Register PrimaryReg = HighReg; 3434 Register SecondaryReg = LowReg; 3435 Register PrimaryShiftReg = Reg32; 3436 Register SecondaryShiftReg = Reg0; 3437 3438 // By default the emitted opcodes check for the set bit from the MSB side. 3439 // Setting SwapPrimarySide checks the set bit from the LSB side 3440 if (SwapPrimarySide) { 3441 PrimaryReg = LowReg; 3442 SecondaryReg = HighReg; 3443 PrimaryShiftReg = Reg0; 3444 SecondaryShiftReg = Reg32; 3445 } 3446 3447 // Check if the primary bits are == -1 3448 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3449 if (!selectOpWithSrcs(BReg, BoolType, I, {PrimaryReg, NegOneReg}, 3450 SPIRV::OpIEqual)) 3451 return false; 3452 3453 // Select secondary bits if true in BReg, otherwise primary bits 3454 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3455 if (!selectOpWithSrcs(TmpReg, ResType, I, {BReg, SecondaryReg, PrimaryReg}, 3456 SelectOp)) 3457 return false; 3458 3459 // 5. Add 32 when high bits are used, otherwise 0 for low bits 3460 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3461 if (!selectOpWithSrcs(ValReg, ResType, I, 3462 {BReg, SecondaryShiftReg, PrimaryShiftReg}, SelectOp)) 3463 return false; 3464 3465 return selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, AddOp); 3466 } 3467 3468 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3469 const SPIRVType *ResType, 3470 MachineInstr &I, 3471 bool IsSigned) const { 3472 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3473 Register OpReg = I.getOperand(2).getReg(); 3474 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3475 // zero or sign extend 3476 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3477 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3478 3479 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3480 case 16: 3481 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3482 case 32: 3483 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3484 case 64: 3485 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3486 /*SwapPrimarySide=*/false); 3487 default: 3488 report_fatal_error( 3489 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3490 } 3491 } 3492 3493 bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg, 3494 const SPIRVType *ResType, 3495 MachineInstr &I) const { 3496 // FindILsb intrinsic only supports 32 bit integers 3497 Register OpReg = I.getOperand(2).getReg(); 3498 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3499 // OpUConvert treats the operand bits as an unsigned i16 and zero extends it 3500 // to an unsigned i32. As this leaves all the least significant bits unchanged 3501 // so the first set bit from the LSB side doesn't change. 3502 unsigned ExtendOpcode = SPIRV::OpUConvert; 3503 unsigned BitSetOpcode = GL::FindILsb; 3504 3505 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3506 case 16: 3507 return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); 3508 case 32: 3509 return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); 3510 case 64: 3511 return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, 3512 /*SwapPrimarySide=*/true); 3513 default: 3514 report_fatal_error("spv_firstbitlow only supports 16,32,64 bits."); 3515 } 3516 } 3517 3518 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3519 const SPIRVType *ResType, 3520 MachineInstr &I) const { 3521 // there was an allocation size parameter to the allocation instruction 3522 // that is not 1 3523 MachineBasicBlock &BB = *I.getParent(); 3524 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3525 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3526 .addDef(ResVReg) 3527 .addUse(GR.getSPIRVTypeID(ResType)) 3528 .addUse(I.getOperand(2).getReg()) 3529 .constrainAllUses(TII, TRI, RBI); 3530 if (!STI.isVulkanEnv()) { 3531 unsigned Alignment = I.getOperand(3).getImm(); 3532 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3533 } 3534 return Res; 3535 } 3536 3537 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3538 const SPIRVType *ResType, 3539 MachineInstr &I) const { 3540 // Change order of instructions if needed: all OpVariable instructions in a 3541 // function must be the first instructions in the first block 3542 auto It = getOpVariableMBBIt(I); 3543 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3544 TII.get(SPIRV::OpVariable)) 3545 .addDef(ResVReg) 3546 .addUse(GR.getSPIRVTypeID(ResType)) 3547 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3548 .constrainAllUses(TII, TRI, RBI); 3549 if (!STI.isVulkanEnv()) { 3550 unsigned Alignment = I.getOperand(2).getImm(); 3551 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3552 {Alignment}); 3553 } 3554 return Res; 3555 } 3556 3557 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3558 // InstructionSelector walks backwards through the instructions. We can use 3559 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3560 // first, so can generate an OpBranchConditional here. If there is no 3561 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3562 const MachineInstr *PrevI = I.getPrevNode(); 3563 MachineBasicBlock &MBB = *I.getParent(); 3564 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3565 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3566 .addUse(PrevI->getOperand(0).getReg()) 3567 .addMBB(PrevI->getOperand(1).getMBB()) 3568 .addMBB(I.getOperand(0).getMBB()) 3569 .constrainAllUses(TII, TRI, RBI); 3570 } 3571 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3572 .addMBB(I.getOperand(0).getMBB()) 3573 .constrainAllUses(TII, TRI, RBI); 3574 } 3575 3576 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3577 // InstructionSelector walks backwards through the instructions. For an 3578 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3579 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3580 // generate the OpBranchConditional in selectBranch above. 3581 // 3582 // If an OpBranchConditional has been generated, we simply return, as the work 3583 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3584 // implicit fallthrough to the next basic block, so we need to create an 3585 // OpBranchConditional with an explicit "false" argument pointing to the next 3586 // basic block that LLVM would fall through to. 3587 const MachineInstr *NextI = I.getNextNode(); 3588 // Check if this has already been successfully selected. 3589 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3590 return true; 3591 // Must be relying on implicit block fallthrough, so generate an 3592 // OpBranchConditional with the "next" basic block as the "false" target. 3593 MachineBasicBlock &MBB = *I.getParent(); 3594 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3595 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3596 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3597 .addUse(I.getOperand(0).getReg()) 3598 .addMBB(I.getOperand(1).getMBB()) 3599 .addMBB(NextMBB) 3600 .constrainAllUses(TII, TRI, RBI); 3601 } 3602 3603 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3604 const SPIRVType *ResType, 3605 MachineInstr &I) const { 3606 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3607 .addDef(ResVReg) 3608 .addUse(GR.getSPIRVTypeID(ResType)); 3609 const unsigned NumOps = I.getNumOperands(); 3610 for (unsigned i = 1; i < NumOps; i += 2) { 3611 MIB.addUse(I.getOperand(i + 0).getReg()); 3612 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3613 } 3614 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3615 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3616 MIB->removeOperand(1); 3617 return Res; 3618 } 3619 3620 bool SPIRVInstructionSelector::selectGlobalValue( 3621 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3622 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3623 MachineIRBuilder MIRBuilder(I); 3624 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3625 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3626 SPIRVType *PointerBaseType; 3627 if (GVType->isArrayTy()) { 3628 SPIRVType *ArrayElementType = 3629 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3630 SPIRV::AccessQualifier::ReadWrite, false); 3631 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3632 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3633 } else { 3634 PointerBaseType = GR.getOrCreateSPIRVType( 3635 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3636 } 3637 3638 std::string GlobalIdent; 3639 if (!GV->hasName()) { 3640 unsigned &ID = UnnamedGlobalIDs[GV]; 3641 if (ID == 0) 3642 ID = UnnamedGlobalIDs.size(); 3643 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3644 } else { 3645 GlobalIdent = GV->getName(); 3646 } 3647 3648 // Behaviour of functions as operands depends on availability of the 3649 // corresponding extension (SPV_INTEL_function_pointers): 3650 // - If there is an extension to operate with functions as operands: 3651 // We create a proper constant operand and evaluate a correct type for a 3652 // function pointer. 3653 // - Without the required extension: 3654 // We have functions as operands in tests with blocks of instruction e.g. in 3655 // transcoding/global_block.ll. These operands are not used and should be 3656 // substituted by zero constants. Their type is expected to be always 3657 // OpTypePointer Function %uchar. 3658 if (isa<Function>(GV)) { 3659 const Constant *ConstVal = GV; 3660 MachineBasicBlock &BB = *I.getParent(); 3661 Register NewReg = GR.find(ConstVal, GR.CurMF); 3662 if (!NewReg.isValid()) { 3663 Register NewReg = ResVReg; 3664 GR.add(ConstVal, GR.CurMF, NewReg); 3665 const Function *GVFun = 3666 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3667 ? dyn_cast<Function>(GV) 3668 : nullptr; 3669 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3670 PointerBaseType, I, TII, 3671 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3672 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3673 if (GVFun) { 3674 // References to a function via function pointers generate virtual 3675 // registers without a definition. We will resolve it later, during 3676 // module analysis stage. 3677 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 3678 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3679 Register FuncVReg = 3680 MRI->createGenericVirtualRegister(GR.getRegType(ResType)); 3681 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); 3682 MachineInstrBuilder MIB1 = 3683 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 3684 .addDef(FuncVReg) 3685 .addUse(ResTypeReg); 3686 MachineInstrBuilder MIB2 = 3687 BuildMI(BB, I, I.getDebugLoc(), 3688 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3689 .addDef(NewReg) 3690 .addUse(ResTypeReg) 3691 .addUse(FuncVReg); 3692 // mapping the function pointer to the used Function 3693 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); 3694 return MIB1.constrainAllUses(TII, TRI, RBI) && 3695 MIB2.constrainAllUses(TII, TRI, RBI); 3696 } 3697 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3698 .addDef(NewReg) 3699 .addUse(GR.getSPIRVTypeID(ResType)) 3700 .constrainAllUses(TII, TRI, RBI); 3701 } 3702 assert(NewReg != ResVReg); 3703 return BuildCOPY(ResVReg, NewReg, I); 3704 } 3705 auto GlobalVar = cast<GlobalVariable>(GV); 3706 assert(GlobalVar->getName() != "llvm.global.annotations"); 3707 3708 // Skip empty declaration for GVs with initializers till we get the decl with 3709 // passed initializer. 3710 if (hasInitializer(GlobalVar) && !Init) 3711 return true; 3712 3713 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage(); 3714 SPIRV::LinkageType::LinkageType LnkType = 3715 GV->isDeclarationForLinker() 3716 ? SPIRV::LinkageType::Import 3717 : (GV->hasLinkOnceODRLinkage() && 3718 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3719 ? SPIRV::LinkageType::LinkOnceODR 3720 : SPIRV::LinkageType::Export); 3721 3722 const unsigned AddrSpace = GV->getAddressSpace(); 3723 SPIRV::StorageClass::StorageClass StorageClass = 3724 addressSpaceToStorageClass(AddrSpace, STI); 3725 SPIRVType *ResType = 3726 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3727 Register Reg = GR.buildGlobalVariable( 3728 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3729 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3730 return Reg.isValid(); 3731 } 3732 3733 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3734 const SPIRVType *ResType, 3735 MachineInstr &I) const { 3736 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3737 return selectExtInst(ResVReg, ResType, I, CL::log10); 3738 } 3739 3740 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3741 // is implemented as: 3742 // log10(x) = log2(x) * (1 / log2(10)) 3743 // = log2(x) * 0.30103 3744 3745 MachineIRBuilder MIRBuilder(I); 3746 MachineBasicBlock &BB = *I.getParent(); 3747 3748 // Build log2(x). 3749 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3750 bool Result = 3751 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3752 .addDef(VarReg) 3753 .addUse(GR.getSPIRVTypeID(ResType)) 3754 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3755 .addImm(GL::Log2) 3756 .add(I.getOperand(1)) 3757 .constrainAllUses(TII, TRI, RBI); 3758 3759 // Build 0.30103. 3760 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3761 ResType->getOpcode() == SPIRV::OpTypeFloat); 3762 // TODO: Add matrix implementation once supported by the HLSL frontend. 3763 const SPIRVType *SpirvScalarType = 3764 ResType->getOpcode() == SPIRV::OpTypeVector 3765 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3766 : ResType; 3767 Register ScaleReg = 3768 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3769 3770 // Multiply log2(x) by 0.30103 to get log10(x) result. 3771 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3772 ? SPIRV::OpVectorTimesScalar 3773 : SPIRV::OpFMulS; 3774 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3775 .addDef(ResVReg) 3776 .addUse(GR.getSPIRVTypeID(ResType)) 3777 .addUse(VarReg) 3778 .addUse(ScaleReg) 3779 .constrainAllUses(TII, TRI, RBI); 3780 } 3781 3782 // Generate the instructions to load 3-element vector builtin input 3783 // IDs/Indices. 3784 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3785 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3786 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3787 const SPIRVType *ResType, MachineInstr &I) const { 3788 MachineIRBuilder MIRBuilder(I); 3789 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3790 const SPIRVType *Vec3Ty = 3791 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3792 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3793 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3794 3795 // Create new register for the input ID builtin variable. 3796 Register NewRegister = 3797 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3798 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3799 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3800 3801 // Build global variable with the necessary decorations for the input ID 3802 // builtin variable. 3803 Register Variable = GR.buildGlobalVariable( 3804 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3805 SPIRV::StorageClass::Input, nullptr, true, true, 3806 SPIRV::LinkageType::Import, MIRBuilder, false); 3807 3808 // Create new register for loading value. 3809 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3810 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3811 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3812 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3813 3814 // Load v3uint value from the global variable. 3815 bool Result = 3816 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3817 .addDef(LoadedRegister) 3818 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3819 .addUse(Variable); 3820 3821 // Get the input ID index. Expecting operand is a constant immediate value, 3822 // wrapped in a type assignment. 3823 assert(I.getOperand(2).isReg()); 3824 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3825 3826 // Extract the input ID from the loaded vector value. 3827 MachineBasicBlock &BB = *I.getParent(); 3828 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3829 .addDef(ResVReg) 3830 .addUse(GR.getSPIRVTypeID(ResType)) 3831 .addUse(LoadedRegister) 3832 .addImm(ThreadId); 3833 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3834 } 3835 3836 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3837 MachineInstr &I) const { 3838 MachineIRBuilder MIRBuilder(I); 3839 if (Type->getOpcode() != SPIRV::OpTypeVector) 3840 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3841 3842 uint64_t VectorSize = Type->getOperand(2).getImm(); 3843 if (VectorSize == 4) 3844 return Type; 3845 3846 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3847 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3848 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3849 } 3850 3851 namespace llvm { 3852 InstructionSelector * 3853 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3854 const SPIRVSubtarget &Subtarget, 3855 const RegisterBankInfo &RBI) { 3856 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3857 } 3858 } // namespace llvm 3859