1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/Register.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/IR/IntrinsicsSPIRV.h" 34 #include "llvm/Support/Debug.h" 35 36 #define DEBUG_TYPE "spirv-isel" 37 38 using namespace llvm; 39 namespace CL = SPIRV::OpenCLExtInst; 40 namespace GL = SPIRV::GLSLExtInst; 41 42 using ExtInstList = 43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 44 45 namespace { 46 47 #define GET_GLOBALISEL_PREDICATE_BITSET 48 #include "SPIRVGenGlobalISel.inc" 49 #undef GET_GLOBALISEL_PREDICATE_BITSET 50 51 class SPIRVInstructionSelector : public InstructionSelector { 52 const SPIRVSubtarget &STI; 53 const SPIRVInstrInfo &TII; 54 const SPIRVRegisterInfo &TRI; 55 const RegisterBankInfo &RBI; 56 SPIRVGlobalRegistry &GR; 57 MachineRegisterInfo *MRI; 58 MachineFunction *HasVRegsReset = nullptr; 59 60 /// We need to keep track of the number we give to anonymous global values to 61 /// generate the same name every time when this is needed. 62 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 63 64 public: 65 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 66 const SPIRVSubtarget &ST, 67 const RegisterBankInfo &RBI); 68 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 69 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 70 BlockFrequencyInfo *BFI) override; 71 // Common selection code. Instruction-specific selection occurs in spvSelect. 72 bool select(MachineInstr &I) override; 73 static const char *getName() { return DEBUG_TYPE; } 74 75 #define GET_GLOBALISEL_PREDICATES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_PREDICATES_DECL 78 79 #define GET_GLOBALISEL_TEMPORARIES_DECL 80 #include "SPIRVGenGlobalISel.inc" 81 #undef GET_GLOBALISEL_TEMPORARIES_DECL 82 83 private: 84 void resetVRegsType(MachineFunction &MF); 85 86 // tblgen-erated 'select' implementation, used as the initial selector for 87 // the patterns that don't require complex C++. 88 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 89 90 // All instruction-specific selection that didn't happen in "select()". 91 // Is basically a large Switch/Case delegating to all other select method. 92 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I) const; 94 95 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 96 MachineInstr &I, bool IsSigned) const; 97 98 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I, bool IsSigned) const; 100 101 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 102 MachineInstr &I, Register SrcReg, 103 bool IsSigned) const; 104 105 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 106 MachineInstr &I, bool IsSigned) const; 107 108 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 109 const MachineInstr *Init = nullptr) const; 110 111 bool selectNAryOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I, std::vector<Register> SrcRegs, 113 unsigned Opcode) const; 114 115 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, Register SrcReg, 117 unsigned Opcode) const; 118 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 119 unsigned Opcode) const; 120 121 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 125 MachineInstr &I) const; 126 bool selectStore(MachineInstr &I) const; 127 128 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 129 MachineInstr &I) const; 130 bool selectStackRestore(MachineInstr &I) const; 131 132 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 133 134 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 135 MachineInstr &I, unsigned NewOpcode, 136 unsigned NegateOpcode = 0) const; 137 138 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 139 MachineInstr &I) const; 140 141 bool selectFence(MachineInstr &I) const; 142 143 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 144 MachineInstr &I) const; 145 146 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 147 MachineInstr &I, unsigned OpType) const; 148 149 bool selectAll(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 152 bool selectAny(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 158 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 163 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 164 unsigned comparisonOpcode, MachineInstr &I) const; 165 bool selectCross(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I) const; 169 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I) const; 171 172 bool selectSign(Register ResVReg, const SPIRVType *ResType, 173 MachineInstr &I) const; 174 175 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 176 MachineInstr &I) const; 177 178 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 179 MachineInstr &I, unsigned Opcode) const; 180 181 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 template <bool Signed> 185 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 template <bool Signed> 188 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 192 int OpIdx) const; 193 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 194 int OpIdx) const; 195 196 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 197 MachineInstr &I) const; 198 199 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 200 bool IsSigned) const; 201 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 202 bool IsSigned, unsigned Opcode) const; 203 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 204 bool IsSigned) const; 205 206 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 207 MachineInstr &I) const; 208 209 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 210 const SPIRVType *intTy, const SPIRVType *boolTy) const; 211 212 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 215 MachineInstr &I) const; 216 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I) const; 218 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I) const; 220 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I) const; 224 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 227 MachineInstr &I) const; 228 229 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 230 MachineInstr &I) const; 231 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 232 MachineInstr &I) const; 233 234 bool selectBranch(MachineInstr &I) const; 235 bool selectBranchCond(MachineInstr &I) const; 236 237 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 240 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 241 MachineInstr &I, GL::GLSLExtInst GLInst) const; 242 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 243 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 244 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 245 MachineInstr &I, CL::OpenCLExtInst CLInst, 246 GL::GLSLExtInst GLInst) const; 247 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 248 MachineInstr &I, const ExtInstList &ExtInsts) const; 249 250 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 251 MachineInstr &I) const; 252 253 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 254 MachineInstr &I) const; 255 256 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 257 MachineInstr &I) const; 258 259 bool selectWaveReadLaneAt(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 262 bool selectUnmergeValues(MachineInstr &I) const; 263 264 void selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 265 MachineInstr &I) const; 266 267 // Utilities 268 Register buildI32Constant(uint32_t Val, MachineInstr &I, 269 const SPIRVType *ResType = nullptr) const; 270 271 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 272 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 273 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 274 MachineInstr &I) const; 275 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 276 277 bool wrapIntoSpecConstantOp(MachineInstr &I, 278 SmallVector<Register> &CompositeArgs) const; 279 280 Register getUcharPtrTypeReg(MachineInstr &I, 281 SPIRV::StorageClass::StorageClass SC) const; 282 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 283 Register Src, Register DestType, 284 uint32_t Opcode) const; 285 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 286 SPIRVType *SrcPtrTy) const; 287 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 288 uint32_t Binding, uint32_t ArraySize, 289 Register IndexReg, bool IsNonUniform, 290 MachineIRBuilder MIRBuilder) const; 291 }; 292 293 } // end anonymous namespace 294 295 #define GET_GLOBALISEL_IMPL 296 #include "SPIRVGenGlobalISel.inc" 297 #undef GET_GLOBALISEL_IMPL 298 299 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 300 const SPIRVSubtarget &ST, 301 const RegisterBankInfo &RBI) 302 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 303 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 304 #define GET_GLOBALISEL_PREDICATES_INIT 305 #include "SPIRVGenGlobalISel.inc" 306 #undef GET_GLOBALISEL_PREDICATES_INIT 307 #define GET_GLOBALISEL_TEMPORARIES_INIT 308 #include "SPIRVGenGlobalISel.inc" 309 #undef GET_GLOBALISEL_TEMPORARIES_INIT 310 { 311 } 312 313 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 314 CodeGenCoverage *CoverageInfo, 315 ProfileSummaryInfo *PSI, 316 BlockFrequencyInfo *BFI) { 317 MRI = &MF.getRegInfo(); 318 GR.setCurrentFunc(MF); 319 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 320 } 321 322 // Ensure that register classes correspond to pattern matching rules. 323 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 324 if (HasVRegsReset == &MF) 325 return; 326 HasVRegsReset = &MF; 327 328 MachineRegisterInfo &MRI = MF.getRegInfo(); 329 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 330 Register Reg = Register::index2VirtReg(I); 331 LLT RegType = MRI.getType(Reg); 332 if (RegType.isScalar()) 333 MRI.setType(Reg, LLT::scalar(64)); 334 else if (RegType.isPointer()) 335 MRI.setType(Reg, LLT::pointer(0, 64)); 336 else if (RegType.isVector()) 337 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 338 } 339 for (const auto &MBB : MF) { 340 for (const auto &MI : MBB) { 341 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 342 continue; 343 Register DstReg = MI.getOperand(0).getReg(); 344 LLT DstType = MRI.getType(DstReg); 345 Register SrcReg = MI.getOperand(1).getReg(); 346 LLT SrcType = MRI.getType(SrcReg); 347 if (DstType != SrcType) 348 MRI.setType(DstReg, MRI.getType(SrcReg)); 349 350 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 351 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 352 if (DstRC != SrcRC && SrcRC) 353 MRI.setRegClass(DstReg, SrcRC); 354 } 355 } 356 } 357 358 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 359 360 // Defined in SPIRVLegalizerInfo.cpp. 361 extern bool isTypeFoldingSupported(unsigned Opcode); 362 363 bool SPIRVInstructionSelector::select(MachineInstr &I) { 364 resetVRegsType(*I.getParent()->getParent()); 365 366 assert(I.getParent() && "Instruction should be in a basic block!"); 367 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 368 369 Register Opcode = I.getOpcode(); 370 // If it's not a GMIR instruction, we've selected it already. 371 if (!isPreISelGenericOpcode(Opcode)) { 372 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 373 Register DstReg = I.getOperand(0).getReg(); 374 Register SrcReg = I.getOperand(1).getReg(); 375 auto *Def = MRI->getVRegDef(SrcReg); 376 if (isTypeFoldingSupported(Def->getOpcode())) { 377 bool Res = selectImpl(I, *CoverageInfo); 378 LLVM_DEBUG({ 379 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 380 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 381 I.print(dbgs()); 382 } 383 }); 384 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 385 if (Res) 386 return Res; 387 } 388 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 389 MRI->replaceRegWith(SrcReg, DstReg); 390 I.removeFromParent(); 391 return true; 392 } else if (I.getNumDefs() == 1) { 393 // Make all vregs 64 bits (for SPIR-V IDs). 394 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 395 } 396 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 397 } 398 399 if (I.getNumOperands() != I.getNumExplicitOperands()) { 400 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 401 return false; 402 } 403 404 // Common code for getting return reg+type, and removing selected instr 405 // from parent occurs here. Instr-specific selection happens in spvSelect(). 406 bool HasDefs = I.getNumDefs() > 0; 407 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 408 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 409 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 410 if (spvSelect(ResVReg, ResType, I)) { 411 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 412 for (unsigned i = 0; i < I.getNumDefs(); ++i) 413 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 414 I.removeFromParent(); 415 return true; 416 } 417 return false; 418 } 419 420 static bool mayApplyGenericSelection(unsigned Opcode) { 421 switch (Opcode) { 422 case TargetOpcode::G_CONSTANT: 423 return false; 424 case TargetOpcode::G_SADDO: 425 case TargetOpcode::G_SSUBO: 426 return true; 427 } 428 return isTypeFoldingSupported(Opcode); 429 } 430 431 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 432 const SPIRVType *ResType, 433 MachineInstr &I) const { 434 const unsigned Opcode = I.getOpcode(); 435 if (mayApplyGenericSelection(Opcode)) 436 return selectImpl(I, *CoverageInfo); 437 switch (Opcode) { 438 case TargetOpcode::G_CONSTANT: 439 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 440 I); 441 case TargetOpcode::G_GLOBAL_VALUE: 442 return selectGlobalValue(ResVReg, I); 443 case TargetOpcode::G_IMPLICIT_DEF: 444 return selectOpUndef(ResVReg, ResType, I); 445 case TargetOpcode::G_FREEZE: 446 return selectFreeze(ResVReg, ResType, I); 447 448 case TargetOpcode::G_INTRINSIC: 449 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 450 case TargetOpcode::G_INTRINSIC_CONVERGENT: 451 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 452 return selectIntrinsic(ResVReg, ResType, I); 453 case TargetOpcode::G_BITREVERSE: 454 return selectBitreverse(ResVReg, ResType, I); 455 456 case TargetOpcode::G_BUILD_VECTOR: 457 return selectBuildVector(ResVReg, ResType, I); 458 case TargetOpcode::G_SPLAT_VECTOR: 459 return selectSplatVector(ResVReg, ResType, I); 460 461 case TargetOpcode::G_SHUFFLE_VECTOR: { 462 MachineBasicBlock &BB = *I.getParent(); 463 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 464 .addDef(ResVReg) 465 .addUse(GR.getSPIRVTypeID(ResType)) 466 .addUse(I.getOperand(1).getReg()) 467 .addUse(I.getOperand(2).getReg()); 468 for (auto V : I.getOperand(3).getShuffleMask()) 469 MIB.addImm(V); 470 return MIB.constrainAllUses(TII, TRI, RBI); 471 } 472 case TargetOpcode::G_MEMMOVE: 473 case TargetOpcode::G_MEMCPY: 474 case TargetOpcode::G_MEMSET: 475 return selectMemOperation(ResVReg, I); 476 477 case TargetOpcode::G_ICMP: 478 return selectICmp(ResVReg, ResType, I); 479 case TargetOpcode::G_FCMP: 480 return selectFCmp(ResVReg, ResType, I); 481 482 case TargetOpcode::G_FRAME_INDEX: 483 return selectFrameIndex(ResVReg, ResType, I); 484 485 case TargetOpcode::G_LOAD: 486 return selectLoad(ResVReg, ResType, I); 487 case TargetOpcode::G_STORE: 488 return selectStore(I); 489 490 case TargetOpcode::G_BR: 491 return selectBranch(I); 492 case TargetOpcode::G_BRCOND: 493 return selectBranchCond(I); 494 495 case TargetOpcode::G_PHI: 496 return selectPhi(ResVReg, ResType, I); 497 498 case TargetOpcode::G_FPTOSI: 499 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 500 case TargetOpcode::G_FPTOUI: 501 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 502 503 case TargetOpcode::G_SITOFP: 504 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 505 case TargetOpcode::G_UITOFP: 506 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 507 508 case TargetOpcode::G_CTPOP: 509 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 510 case TargetOpcode::G_SMIN: 511 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 512 case TargetOpcode::G_UMIN: 513 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 514 515 case TargetOpcode::G_SMAX: 516 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 517 case TargetOpcode::G_UMAX: 518 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 519 520 case TargetOpcode::G_FMA: 521 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 522 523 case TargetOpcode::G_FPOW: 524 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 525 case TargetOpcode::G_FPOWI: 526 return selectExtInst(ResVReg, ResType, I, CL::pown); 527 528 case TargetOpcode::G_FEXP: 529 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 530 case TargetOpcode::G_FEXP2: 531 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 532 533 case TargetOpcode::G_FLOG: 534 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 535 case TargetOpcode::G_FLOG2: 536 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 537 case TargetOpcode::G_FLOG10: 538 return selectLog10(ResVReg, ResType, I); 539 540 case TargetOpcode::G_FABS: 541 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 542 case TargetOpcode::G_ABS: 543 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 544 545 case TargetOpcode::G_FMINNUM: 546 case TargetOpcode::G_FMINIMUM: 547 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 548 case TargetOpcode::G_FMAXNUM: 549 case TargetOpcode::G_FMAXIMUM: 550 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 551 552 case TargetOpcode::G_FCOPYSIGN: 553 return selectExtInst(ResVReg, ResType, I, CL::copysign); 554 555 case TargetOpcode::G_FCEIL: 556 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 557 case TargetOpcode::G_FFLOOR: 558 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 559 560 case TargetOpcode::G_FCOS: 561 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 562 case TargetOpcode::G_FSIN: 563 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 564 case TargetOpcode::G_FTAN: 565 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 566 case TargetOpcode::G_FACOS: 567 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 568 case TargetOpcode::G_FASIN: 569 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 570 case TargetOpcode::G_FATAN: 571 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 572 case TargetOpcode::G_FATAN2: 573 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 574 case TargetOpcode::G_FCOSH: 575 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 576 case TargetOpcode::G_FSINH: 577 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 578 case TargetOpcode::G_FTANH: 579 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 580 581 case TargetOpcode::G_FSQRT: 582 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 583 584 case TargetOpcode::G_CTTZ: 585 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 586 return selectExtInst(ResVReg, ResType, I, CL::ctz); 587 case TargetOpcode::G_CTLZ: 588 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 589 return selectExtInst(ResVReg, ResType, I, CL::clz); 590 591 case TargetOpcode::G_INTRINSIC_ROUND: 592 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 593 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 594 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 595 case TargetOpcode::G_INTRINSIC_TRUNC: 596 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 597 case TargetOpcode::G_FRINT: 598 case TargetOpcode::G_FNEARBYINT: 599 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 600 601 case TargetOpcode::G_SMULH: 602 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 603 case TargetOpcode::G_UMULH: 604 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 605 606 case TargetOpcode::G_SADDSAT: 607 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 608 case TargetOpcode::G_UADDSAT: 609 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 610 case TargetOpcode::G_SSUBSAT: 611 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 612 case TargetOpcode::G_USUBSAT: 613 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 614 615 case TargetOpcode::G_UADDO: 616 return selectOverflowArith(ResVReg, ResType, I, 617 ResType->getOpcode() == SPIRV::OpTypeVector 618 ? SPIRV::OpIAddCarryV 619 : SPIRV::OpIAddCarryS); 620 case TargetOpcode::G_USUBO: 621 return selectOverflowArith(ResVReg, ResType, I, 622 ResType->getOpcode() == SPIRV::OpTypeVector 623 ? SPIRV::OpISubBorrowV 624 : SPIRV::OpISubBorrowS); 625 case TargetOpcode::G_UMULO: 626 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 627 case TargetOpcode::G_SMULO: 628 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 629 630 case TargetOpcode::G_SEXT: 631 return selectExt(ResVReg, ResType, I, true); 632 case TargetOpcode::G_ANYEXT: 633 case TargetOpcode::G_ZEXT: 634 return selectExt(ResVReg, ResType, I, false); 635 case TargetOpcode::G_TRUNC: 636 return selectTrunc(ResVReg, ResType, I); 637 case TargetOpcode::G_FPTRUNC: 638 case TargetOpcode::G_FPEXT: 639 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 640 641 case TargetOpcode::G_PTRTOINT: 642 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 643 case TargetOpcode::G_INTTOPTR: 644 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 645 case TargetOpcode::G_BITCAST: 646 return selectBitcast(ResVReg, ResType, I); 647 case TargetOpcode::G_ADDRSPACE_CAST: 648 return selectAddrSpaceCast(ResVReg, ResType, I); 649 case TargetOpcode::G_PTR_ADD: { 650 // Currently, we get G_PTR_ADD only applied to global variables. 651 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 652 Register GV = I.getOperand(1).getReg(); 653 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 654 (void)II; 655 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 656 (*II).getOpcode() == TargetOpcode::COPY || 657 (*II).getOpcode() == SPIRV::OpVariable) && 658 isImm(I.getOperand(2), MRI)); 659 // It may be the initialization of a global variable. 660 bool IsGVInit = false; 661 for (MachineRegisterInfo::use_instr_iterator 662 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 663 UseEnd = MRI->use_instr_end(); 664 UseIt != UseEnd; UseIt = std::next(UseIt)) { 665 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 666 (*UseIt).getOpcode() == SPIRV::OpVariable) { 667 IsGVInit = true; 668 break; 669 } 670 } 671 MachineBasicBlock &BB = *I.getParent(); 672 if (!IsGVInit) { 673 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 674 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 675 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 676 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 677 // Build a new virtual register that is associated with the required 678 // data type. 679 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 680 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 681 // Having a correctly typed base we are ready to build the actually 682 // required GEP. It may not be a constant though, because all Operands 683 // of OpSpecConstantOp is to originate from other const instructions, 684 // and only the AccessChain named opcodes accept a global OpVariable 685 // instruction. We can't use an AccessChain opcode because of the type 686 // mismatch between result and base types. 687 if (!GR.isBitcastCompatible(ResType, GVType)) 688 report_fatal_error( 689 "incompatible result and operand types in a bitcast"); 690 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 691 MachineInstrBuilder MIB = 692 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 693 .addDef(NewVReg) 694 .addUse(ResTypeReg) 695 .addUse(GV); 696 return MIB.constrainAllUses(TII, TRI, RBI) && 697 BuildMI(BB, I, I.getDebugLoc(), 698 TII.get(STI.isVulkanEnv() 699 ? SPIRV::OpInBoundsAccessChain 700 : SPIRV::OpInBoundsPtrAccessChain)) 701 .addDef(ResVReg) 702 .addUse(ResTypeReg) 703 .addUse(NewVReg) 704 .addUse(I.getOperand(2).getReg()) 705 .constrainAllUses(TII, TRI, RBI); 706 } else { 707 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 708 .addDef(ResVReg) 709 .addUse(GR.getSPIRVTypeID(ResType)) 710 .addImm( 711 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 712 .addUse(GV) 713 .addUse(I.getOperand(2).getReg()) 714 .constrainAllUses(TII, TRI, RBI); 715 } 716 } 717 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 718 // initialize a global variable with a constant expression (e.g., the test 719 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 720 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 721 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 722 .addDef(ResVReg) 723 .addUse(GR.getSPIRVTypeID(ResType)) 724 .addImm(static_cast<uint32_t>( 725 SPIRV::Opcode::InBoundsPtrAccessChain)) 726 .addUse(GV) 727 .addUse(Idx) 728 .addUse(I.getOperand(2).getReg()); 729 return MIB.constrainAllUses(TII, TRI, RBI); 730 } 731 732 case TargetOpcode::G_ATOMICRMW_OR: 733 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 734 case TargetOpcode::G_ATOMICRMW_ADD: 735 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 736 case TargetOpcode::G_ATOMICRMW_AND: 737 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 738 case TargetOpcode::G_ATOMICRMW_MAX: 739 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 740 case TargetOpcode::G_ATOMICRMW_MIN: 741 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 742 case TargetOpcode::G_ATOMICRMW_SUB: 743 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 744 case TargetOpcode::G_ATOMICRMW_XOR: 745 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 746 case TargetOpcode::G_ATOMICRMW_UMAX: 747 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 748 case TargetOpcode::G_ATOMICRMW_UMIN: 749 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 750 case TargetOpcode::G_ATOMICRMW_XCHG: 751 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 752 case TargetOpcode::G_ATOMIC_CMPXCHG: 753 return selectAtomicCmpXchg(ResVReg, ResType, I); 754 755 case TargetOpcode::G_ATOMICRMW_FADD: 756 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 757 case TargetOpcode::G_ATOMICRMW_FSUB: 758 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 759 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 760 SPIRV::OpFNegate); 761 case TargetOpcode::G_ATOMICRMW_FMIN: 762 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 763 case TargetOpcode::G_ATOMICRMW_FMAX: 764 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 765 766 case TargetOpcode::G_FENCE: 767 return selectFence(I); 768 769 case TargetOpcode::G_STACKSAVE: 770 return selectStackSave(ResVReg, ResType, I); 771 case TargetOpcode::G_STACKRESTORE: 772 return selectStackRestore(I); 773 774 case TargetOpcode::G_UNMERGE_VALUES: 775 return selectUnmergeValues(I); 776 777 // Discard gen opcodes for intrinsics which we do not expect to actually 778 // represent code after lowering or intrinsics which are not implemented but 779 // should not crash when found in a customer's LLVM IR input. 780 case TargetOpcode::G_TRAP: 781 case TargetOpcode::G_DEBUGTRAP: 782 case TargetOpcode::G_UBSANTRAP: 783 case TargetOpcode::DBG_LABEL: 784 return true; 785 786 default: 787 return false; 788 } 789 } 790 791 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 792 const SPIRVType *ResType, 793 MachineInstr &I, 794 GL::GLSLExtInst GLInst) const { 795 return selectExtInst(ResVReg, ResType, I, 796 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 797 } 798 799 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 800 const SPIRVType *ResType, 801 MachineInstr &I, 802 CL::OpenCLExtInst CLInst) const { 803 return selectExtInst(ResVReg, ResType, I, 804 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 805 } 806 807 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 808 const SPIRVType *ResType, 809 MachineInstr &I, 810 CL::OpenCLExtInst CLInst, 811 GL::GLSLExtInst GLInst) const { 812 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 813 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 814 return selectExtInst(ResVReg, ResType, I, ExtInsts); 815 } 816 817 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 818 const SPIRVType *ResType, 819 MachineInstr &I, 820 const ExtInstList &Insts) const { 821 822 for (const auto &Ex : Insts) { 823 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 824 uint32_t Opcode = Ex.second; 825 if (STI.canUseExtInstSet(Set)) { 826 MachineBasicBlock &BB = *I.getParent(); 827 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 828 .addDef(ResVReg) 829 .addUse(GR.getSPIRVTypeID(ResType)) 830 .addImm(static_cast<uint32_t>(Set)) 831 .addImm(Opcode); 832 const unsigned NumOps = I.getNumOperands(); 833 unsigned Index = 1; 834 if (Index < NumOps && 835 I.getOperand(Index).getType() == 836 MachineOperand::MachineOperandType::MO_IntrinsicID) 837 Index = 2; 838 for (; Index < NumOps; ++Index) 839 MIB.add(I.getOperand(Index)); 840 return MIB.constrainAllUses(TII, TRI, RBI); 841 } 842 } 843 return false; 844 } 845 846 bool SPIRVInstructionSelector::selectNAryOpWithSrcs(Register ResVReg, 847 const SPIRVType *ResType, 848 MachineInstr &I, 849 std::vector<Register> Srcs, 850 unsigned Opcode) const { 851 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 852 .addDef(ResVReg) 853 .addUse(GR.getSPIRVTypeID(ResType)); 854 for (Register SReg : Srcs) { 855 MIB.addUse(SReg); 856 } 857 return MIB.constrainAllUses(TII, TRI, RBI); 858 } 859 860 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 861 const SPIRVType *ResType, 862 MachineInstr &I, 863 Register SrcReg, 864 unsigned Opcode) const { 865 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 866 .addDef(ResVReg) 867 .addUse(GR.getSPIRVTypeID(ResType)) 868 .addUse(SrcReg) 869 .constrainAllUses(TII, TRI, RBI); 870 } 871 872 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 873 const SPIRVType *ResType, 874 MachineInstr &I, 875 unsigned Opcode) const { 876 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 877 Register SrcReg = I.getOperand(1).getReg(); 878 bool IsGV = false; 879 for (MachineRegisterInfo::def_instr_iterator DefIt = 880 MRI->def_instr_begin(SrcReg); 881 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 882 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 883 IsGV = true; 884 break; 885 } 886 } 887 if (IsGV) { 888 uint32_t SpecOpcode = 0; 889 switch (Opcode) { 890 case SPIRV::OpConvertPtrToU: 891 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 892 break; 893 case SPIRV::OpConvertUToPtr: 894 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 895 break; 896 } 897 if (SpecOpcode) 898 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 899 TII.get(SPIRV::OpSpecConstantOp)) 900 .addDef(ResVReg) 901 .addUse(GR.getSPIRVTypeID(ResType)) 902 .addImm(SpecOpcode) 903 .addUse(SrcReg) 904 .constrainAllUses(TII, TRI, RBI); 905 } 906 } 907 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 908 Opcode); 909 } 910 911 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 912 const SPIRVType *ResType, 913 MachineInstr &I) const { 914 Register OpReg = I.getOperand(1).getReg(); 915 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 916 if (!GR.isBitcastCompatible(ResType, OpType)) 917 report_fatal_error("incompatible result and operand types in a bitcast"); 918 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 919 } 920 921 static void addMemoryOperands(MachineMemOperand *MemOp, 922 MachineInstrBuilder &MIB) { 923 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 924 if (MemOp->isVolatile()) 925 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 926 if (MemOp->isNonTemporal()) 927 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 928 if (MemOp->getAlign().value()) 929 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 930 931 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 932 MIB.addImm(SpvMemOp); 933 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 934 MIB.addImm(MemOp->getAlign().value()); 935 } 936 } 937 938 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 939 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 940 if (Flags & MachineMemOperand::Flags::MOVolatile) 941 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 942 if (Flags & MachineMemOperand::Flags::MONonTemporal) 943 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 944 945 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 946 MIB.addImm(SpvMemOp); 947 } 948 949 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 950 const SPIRVType *ResType, 951 MachineInstr &I) const { 952 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 953 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 954 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 955 .addDef(ResVReg) 956 .addUse(GR.getSPIRVTypeID(ResType)) 957 .addUse(Ptr); 958 if (!I.getNumMemOperands()) { 959 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 960 I.getOpcode() == 961 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 962 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 963 } else { 964 addMemoryOperands(*I.memoperands_begin(), MIB); 965 } 966 return MIB.constrainAllUses(TII, TRI, RBI); 967 } 968 969 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 970 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 971 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 972 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 973 MachineBasicBlock &BB = *I.getParent(); 974 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 975 .addUse(Ptr) 976 .addUse(StoreVal); 977 if (!I.getNumMemOperands()) { 978 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 979 I.getOpcode() == 980 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 981 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 982 } else { 983 addMemoryOperands(*I.memoperands_begin(), MIB); 984 } 985 return MIB.constrainAllUses(TII, TRI, RBI); 986 } 987 988 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 989 const SPIRVType *ResType, 990 MachineInstr &I) const { 991 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 992 report_fatal_error( 993 "llvm.stacksave intrinsic: this instruction requires the following " 994 "SPIR-V extension: SPV_INTEL_variable_length_array", 995 false); 996 MachineBasicBlock &BB = *I.getParent(); 997 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 998 .addDef(ResVReg) 999 .addUse(GR.getSPIRVTypeID(ResType)) 1000 .constrainAllUses(TII, TRI, RBI); 1001 } 1002 1003 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1004 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1005 report_fatal_error( 1006 "llvm.stackrestore intrinsic: this instruction requires the following " 1007 "SPIR-V extension: SPV_INTEL_variable_length_array", 1008 false); 1009 if (!I.getOperand(0).isReg()) 1010 return false; 1011 MachineBasicBlock &BB = *I.getParent(); 1012 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1013 .addUse(I.getOperand(0).getReg()) 1014 .constrainAllUses(TII, TRI, RBI); 1015 } 1016 1017 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1018 MachineInstr &I) const { 1019 MachineBasicBlock &BB = *I.getParent(); 1020 Register SrcReg = I.getOperand(1).getReg(); 1021 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1022 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1023 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1024 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1025 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1026 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1027 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1028 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1029 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1030 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1031 Function &CurFunction = GR.CurMF->getFunction(); 1032 Type *LLVMArrTy = 1033 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1034 // Module takes ownership of the global var. 1035 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1036 true, GlobalValue::InternalLinkage, 1037 Constant::getNullValue(LLVMArrTy)); 1038 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1039 GR.add(GV, GR.CurMF, VarReg); 1040 1041 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1042 .addDef(VarReg) 1043 .addUse(GR.getSPIRVTypeID(VarTy)) 1044 .addImm(SPIRV::StorageClass::UniformConstant) 1045 .addUse(Const) 1046 .constrainAllUses(TII, TRI, RBI); 1047 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1048 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1049 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1050 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1051 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 1052 } 1053 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1054 .addUse(I.getOperand(0).getReg()) 1055 .addUse(SrcReg) 1056 .addUse(I.getOperand(2).getReg()); 1057 if (I.getNumMemOperands()) 1058 addMemoryOperands(*I.memoperands_begin(), MIB); 1059 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1060 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1061 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1062 .addUse(MIB->getOperand(0).getReg()); 1063 return Result; 1064 } 1065 1066 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1067 const SPIRVType *ResType, 1068 MachineInstr &I, 1069 unsigned NewOpcode, 1070 unsigned NegateOpcode) const { 1071 assert(I.hasOneMemOperand()); 1072 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1073 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1074 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1075 Register ScopeReg = buildI32Constant(Scope, I); 1076 1077 Register Ptr = I.getOperand(1).getReg(); 1078 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1079 // auto ScSem = 1080 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1081 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1082 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1083 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 1084 1085 bool Result = false; 1086 Register ValueReg = I.getOperand(2).getReg(); 1087 if (NegateOpcode != 0) { 1088 // Translation with negative value operand is requested 1089 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1090 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1091 ValueReg = TmpReg; 1092 } 1093 1094 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1095 .addDef(ResVReg) 1096 .addUse(GR.getSPIRVTypeID(ResType)) 1097 .addUse(Ptr) 1098 .addUse(ScopeReg) 1099 .addUse(MemSemReg) 1100 .addUse(ValueReg) 1101 .constrainAllUses(TII, TRI, RBI); 1102 return Result; 1103 } 1104 1105 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1106 unsigned ArgI = I.getNumOperands() - 1; 1107 Register SrcReg = 1108 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1109 SPIRVType *DefType = 1110 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1111 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1112 report_fatal_error( 1113 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1114 1115 SPIRVType *ScalarType = 1116 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1117 MachineBasicBlock &BB = *I.getParent(); 1118 bool Res = false; 1119 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1120 Register ResVReg = I.getOperand(i).getReg(); 1121 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1122 if (!ResType) { 1123 // There was no "assign type" actions, let's fix this now 1124 ResType = ScalarType; 1125 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1126 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1127 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1128 } 1129 auto MIB = 1130 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1131 .addDef(ResVReg) 1132 .addUse(GR.getSPIRVTypeID(ResType)) 1133 .addUse(SrcReg) 1134 .addImm(static_cast<int64_t>(i)); 1135 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1136 } 1137 return Res; 1138 } 1139 1140 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1141 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1142 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1143 Register MemSemReg = buildI32Constant(MemSem, I); 1144 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1145 uint32_t Scope = static_cast<uint32_t>( 1146 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1147 Register ScopeReg = buildI32Constant(Scope, I); 1148 MachineBasicBlock &BB = *I.getParent(); 1149 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1150 .addUse(ScopeReg) 1151 .addUse(MemSemReg) 1152 .constrainAllUses(TII, TRI, RBI); 1153 } 1154 1155 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1156 const SPIRVType *ResType, 1157 MachineInstr &I, 1158 unsigned Opcode) const { 1159 Type *ResTy = nullptr; 1160 StringRef ResName; 1161 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1162 report_fatal_error( 1163 "Not enough info to select the arithmetic with overflow instruction"); 1164 if (!ResTy || !ResTy->isStructTy()) 1165 report_fatal_error("Expect struct type result for the arithmetic " 1166 "with overflow instruction"); 1167 // "Result Type must be from OpTypeStruct. The struct must have two members, 1168 // and the two members must be the same type." 1169 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1170 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1171 // Build SPIR-V types and constant(s) if needed. 1172 MachineIRBuilder MIRBuilder(I); 1173 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1174 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1175 assert(I.getNumDefs() > 1 && "Not enought operands"); 1176 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1177 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1178 if (N > 1) 1179 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1180 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1181 Register ZeroReg = buildZerosVal(ResType, I); 1182 // A new virtual register to store the result struct. 1183 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1184 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1185 // Build the result name if needed. 1186 if (ResName.size() > 0) 1187 buildOpName(StructVReg, ResName, MIRBuilder); 1188 // Build the arithmetic with overflow instruction. 1189 MachineBasicBlock &BB = *I.getParent(); 1190 auto MIB = 1191 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1192 .addDef(StructVReg) 1193 .addUse(GR.getSPIRVTypeID(StructType)); 1194 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1195 MIB.addUse(I.getOperand(i).getReg()); 1196 bool Status = MIB.constrainAllUses(TII, TRI, RBI); 1197 // Build instructions to extract fields of the instruction's result. 1198 // A new virtual register to store the higher part of the result struct. 1199 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1200 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1201 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1202 auto MIB = 1203 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1204 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1205 .addUse(GR.getSPIRVTypeID(ResType)) 1206 .addUse(StructVReg) 1207 .addImm(i); 1208 Status &= MIB.constrainAllUses(TII, TRI, RBI); 1209 } 1210 // Build boolean value from the higher part. 1211 Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1212 .addDef(I.getOperand(1).getReg()) 1213 .addUse(BoolTypeReg) 1214 .addUse(HigherVReg) 1215 .addUse(ZeroReg) 1216 .constrainAllUses(TII, TRI, RBI); 1217 return Status; 1218 } 1219 1220 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1221 const SPIRVType *ResType, 1222 MachineInstr &I) const { 1223 Register ScopeReg; 1224 Register MemSemEqReg; 1225 Register MemSemNeqReg; 1226 Register Ptr = I.getOperand(2).getReg(); 1227 if (!isa<GIntrinsic>(I)) { 1228 assert(I.hasOneMemOperand()); 1229 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1230 unsigned Scope = static_cast<uint32_t>(getMemScope( 1231 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1232 ScopeReg = buildI32Constant(Scope, I); 1233 1234 unsigned ScSem = static_cast<uint32_t>( 1235 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1236 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1237 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1238 MemSemEqReg = buildI32Constant(MemSemEq, I); 1239 AtomicOrdering FO = MemOp->getFailureOrdering(); 1240 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1241 MemSemNeqReg = 1242 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 1243 } else { 1244 ScopeReg = I.getOperand(5).getReg(); 1245 MemSemEqReg = I.getOperand(6).getReg(); 1246 MemSemNeqReg = I.getOperand(7).getReg(); 1247 } 1248 1249 Register Cmp = I.getOperand(3).getReg(); 1250 Register Val = I.getOperand(4).getReg(); 1251 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1252 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1253 const DebugLoc &DL = I.getDebugLoc(); 1254 bool Result = 1255 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1256 .addDef(ACmpRes) 1257 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1258 .addUse(Ptr) 1259 .addUse(ScopeReg) 1260 .addUse(MemSemEqReg) 1261 .addUse(MemSemNeqReg) 1262 .addUse(Val) 1263 .addUse(Cmp) 1264 .constrainAllUses(TII, TRI, RBI); 1265 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1266 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1267 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1268 .addDef(CmpSuccReg) 1269 .addUse(GR.getSPIRVTypeID(BoolTy)) 1270 .addUse(ACmpRes) 1271 .addUse(Cmp) 1272 .constrainAllUses(TII, TRI, RBI); 1273 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1274 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1275 .addDef(TmpReg) 1276 .addUse(GR.getSPIRVTypeID(ResType)) 1277 .addUse(ACmpRes) 1278 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1279 .addImm(0) 1280 .constrainAllUses(TII, TRI, RBI); 1281 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1282 .addDef(ResVReg) 1283 .addUse(GR.getSPIRVTypeID(ResType)) 1284 .addUse(CmpSuccReg) 1285 .addUse(TmpReg) 1286 .addImm(1) 1287 .constrainAllUses(TII, TRI, RBI); 1288 return Result; 1289 } 1290 1291 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1292 switch (SC) { 1293 case SPIRV::StorageClass::Workgroup: 1294 case SPIRV::StorageClass::CrossWorkgroup: 1295 case SPIRV::StorageClass::Function: 1296 return true; 1297 default: 1298 return false; 1299 } 1300 } 1301 1302 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1303 switch (SC) { 1304 case SPIRV::StorageClass::DeviceOnlyINTEL: 1305 case SPIRV::StorageClass::HostOnlyINTEL: 1306 return true; 1307 default: 1308 return false; 1309 } 1310 } 1311 1312 // Returns true ResVReg is referred only from global vars and OpName's. 1313 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1314 bool IsGRef = false; 1315 bool IsAllowedRefs = 1316 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1317 [&IsGRef](auto const &It) { 1318 unsigned Opcode = It.getOpcode(); 1319 if (Opcode == SPIRV::OpConstantComposite || 1320 Opcode == SPIRV::OpVariable || 1321 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1322 return IsGRef = true; 1323 return Opcode == SPIRV::OpName; 1324 }); 1325 return IsAllowedRefs && IsGRef; 1326 } 1327 1328 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1329 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1330 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1331 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1332 } 1333 1334 MachineInstrBuilder 1335 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1336 Register Src, Register DestType, 1337 uint32_t Opcode) const { 1338 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1339 TII.get(SPIRV::OpSpecConstantOp)) 1340 .addDef(Dest) 1341 .addUse(DestType) 1342 .addImm(Opcode) 1343 .addUse(Src); 1344 } 1345 1346 MachineInstrBuilder 1347 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1348 SPIRVType *SrcPtrTy) const { 1349 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1350 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1351 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1352 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1353 SPIRV::StorageClass::Generic), 1354 GR.getPointerSize())); 1355 MachineFunction *MF = I.getParent()->getParent(); 1356 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1357 MachineInstrBuilder MIB = buildSpecConstantOp( 1358 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1359 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1360 GR.add(MIB.getInstr(), MF, Tmp); 1361 return MIB; 1362 } 1363 1364 // In SPIR-V address space casting can only happen to and from the Generic 1365 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1366 // pointers to and from Generic pointers. As such, we can convert e.g. from 1367 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1368 // other combinations can only be done by a bitcast, and are probably not safe. 1369 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1370 const SPIRVType *ResType, 1371 MachineInstr &I) const { 1372 MachineBasicBlock &BB = *I.getParent(); 1373 const DebugLoc &DL = I.getDebugLoc(); 1374 1375 Register SrcPtr = I.getOperand(1).getReg(); 1376 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1377 1378 // don't generate a cast for a null that may be represented by OpTypeInt 1379 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1380 ResType->getOpcode() != SPIRV::OpTypePointer) 1381 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1382 .addDef(ResVReg) 1383 .addUse(SrcPtr) 1384 .constrainAllUses(TII, TRI, RBI); 1385 1386 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1387 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1388 1389 if (isASCastInGVar(MRI, ResVReg)) { 1390 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1391 // are expressed by OpSpecConstantOp with an Opcode. 1392 // TODO: maybe insert a check whether the Kernel capability was declared and 1393 // so PtrCastToGeneric/GenericCastToPtr are available. 1394 unsigned SpecOpcode = 1395 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1396 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1397 : (SrcSC == SPIRV::StorageClass::Generic && 1398 isGenericCastablePtr(DstSC) 1399 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1400 : 0); 1401 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1402 // correct value of ResType and use general i8* instead. Maybe this should 1403 // be addressed in the emit-intrinsic step to infer a correct 1404 // OpConstantComposite type. 1405 if (SpecOpcode) { 1406 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1407 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1408 .constrainAllUses(TII, TRI, RBI); 1409 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1410 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1411 return MIB.constrainAllUses(TII, TRI, RBI) && 1412 buildSpecConstantOp( 1413 I, ResVReg, MIB->getOperand(0).getReg(), 1414 getUcharPtrTypeReg(I, DstSC), 1415 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1416 .constrainAllUses(TII, TRI, RBI); 1417 } 1418 } 1419 1420 // don't generate a cast between identical storage classes 1421 if (SrcSC == DstSC) 1422 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1423 .addDef(ResVReg) 1424 .addUse(SrcPtr) 1425 .constrainAllUses(TII, TRI, RBI); 1426 1427 // Casting from an eligible pointer to Generic. 1428 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1429 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1430 // Casting from Generic to an eligible pointer. 1431 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1432 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1433 // Casting between 2 eligible pointers using Generic as an intermediary. 1434 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1435 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1436 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1437 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1438 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1439 .addDef(Tmp) 1440 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1441 .addUse(SrcPtr) 1442 .constrainAllUses(TII, TRI, RBI); 1443 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1444 .addDef(ResVReg) 1445 .addUse(GR.getSPIRVTypeID(ResType)) 1446 .addUse(Tmp) 1447 .constrainAllUses(TII, TRI, RBI); 1448 } 1449 1450 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1451 // be applied 1452 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1453 return selectUnOp(ResVReg, ResType, I, 1454 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1455 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1456 return selectUnOp(ResVReg, ResType, I, 1457 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1458 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1459 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1460 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1461 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1462 1463 // Bitcast for pointers requires that the address spaces must match 1464 return false; 1465 } 1466 1467 static unsigned getFCmpOpcode(unsigned PredNum) { 1468 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1469 switch (Pred) { 1470 case CmpInst::FCMP_OEQ: 1471 return SPIRV::OpFOrdEqual; 1472 case CmpInst::FCMP_OGE: 1473 return SPIRV::OpFOrdGreaterThanEqual; 1474 case CmpInst::FCMP_OGT: 1475 return SPIRV::OpFOrdGreaterThan; 1476 case CmpInst::FCMP_OLE: 1477 return SPIRV::OpFOrdLessThanEqual; 1478 case CmpInst::FCMP_OLT: 1479 return SPIRV::OpFOrdLessThan; 1480 case CmpInst::FCMP_ONE: 1481 return SPIRV::OpFOrdNotEqual; 1482 case CmpInst::FCMP_ORD: 1483 return SPIRV::OpOrdered; 1484 case CmpInst::FCMP_UEQ: 1485 return SPIRV::OpFUnordEqual; 1486 case CmpInst::FCMP_UGE: 1487 return SPIRV::OpFUnordGreaterThanEqual; 1488 case CmpInst::FCMP_UGT: 1489 return SPIRV::OpFUnordGreaterThan; 1490 case CmpInst::FCMP_ULE: 1491 return SPIRV::OpFUnordLessThanEqual; 1492 case CmpInst::FCMP_ULT: 1493 return SPIRV::OpFUnordLessThan; 1494 case CmpInst::FCMP_UNE: 1495 return SPIRV::OpFUnordNotEqual; 1496 case CmpInst::FCMP_UNO: 1497 return SPIRV::OpUnordered; 1498 default: 1499 llvm_unreachable("Unknown predicate type for FCmp"); 1500 } 1501 } 1502 1503 static unsigned getICmpOpcode(unsigned PredNum) { 1504 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1505 switch (Pred) { 1506 case CmpInst::ICMP_EQ: 1507 return SPIRV::OpIEqual; 1508 case CmpInst::ICMP_NE: 1509 return SPIRV::OpINotEqual; 1510 case CmpInst::ICMP_SGE: 1511 return SPIRV::OpSGreaterThanEqual; 1512 case CmpInst::ICMP_SGT: 1513 return SPIRV::OpSGreaterThan; 1514 case CmpInst::ICMP_SLE: 1515 return SPIRV::OpSLessThanEqual; 1516 case CmpInst::ICMP_SLT: 1517 return SPIRV::OpSLessThan; 1518 case CmpInst::ICMP_UGE: 1519 return SPIRV::OpUGreaterThanEqual; 1520 case CmpInst::ICMP_UGT: 1521 return SPIRV::OpUGreaterThan; 1522 case CmpInst::ICMP_ULE: 1523 return SPIRV::OpULessThanEqual; 1524 case CmpInst::ICMP_ULT: 1525 return SPIRV::OpULessThan; 1526 default: 1527 llvm_unreachable("Unknown predicate type for ICmp"); 1528 } 1529 } 1530 1531 static unsigned getPtrCmpOpcode(unsigned Pred) { 1532 switch (static_cast<CmpInst::Predicate>(Pred)) { 1533 case CmpInst::ICMP_EQ: 1534 return SPIRV::OpPtrEqual; 1535 case CmpInst::ICMP_NE: 1536 return SPIRV::OpPtrNotEqual; 1537 default: 1538 llvm_unreachable("Unknown predicate type for pointer comparison"); 1539 } 1540 } 1541 1542 // Return the logical operation, or abort if none exists. 1543 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1544 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1545 switch (Pred) { 1546 case CmpInst::ICMP_EQ: 1547 return SPIRV::OpLogicalEqual; 1548 case CmpInst::ICMP_NE: 1549 return SPIRV::OpLogicalNotEqual; 1550 default: 1551 llvm_unreachable("Unknown predicate type for Bool comparison"); 1552 } 1553 } 1554 1555 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1556 if (!LLVMFloatTy) 1557 return APFloat::getZero(APFloat::IEEEsingle()); 1558 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1559 case Type::HalfTyID: 1560 return APFloat::getZero(APFloat::IEEEhalf()); 1561 default: 1562 case Type::FloatTyID: 1563 return APFloat::getZero(APFloat::IEEEsingle()); 1564 case Type::DoubleTyID: 1565 return APFloat::getZero(APFloat::IEEEdouble()); 1566 } 1567 } 1568 1569 static APFloat getOneFP(const Type *LLVMFloatTy) { 1570 if (!LLVMFloatTy) 1571 return APFloat::getOne(APFloat::IEEEsingle()); 1572 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1573 case Type::HalfTyID: 1574 return APFloat::getOne(APFloat::IEEEhalf()); 1575 default: 1576 case Type::FloatTyID: 1577 return APFloat::getOne(APFloat::IEEEsingle()); 1578 case Type::DoubleTyID: 1579 return APFloat::getOne(APFloat::IEEEdouble()); 1580 } 1581 } 1582 1583 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1584 const SPIRVType *ResType, 1585 MachineInstr &I, 1586 unsigned OpAnyOrAll) const { 1587 assert(I.getNumOperands() == 3); 1588 assert(I.getOperand(2).isReg()); 1589 MachineBasicBlock &BB = *I.getParent(); 1590 Register InputRegister = I.getOperand(2).getReg(); 1591 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1592 1593 if (!InputType) 1594 report_fatal_error("Input Type could not be determined."); 1595 1596 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1597 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1598 if (IsBoolTy && !IsVectorTy) { 1599 assert(ResVReg == I.getOperand(0).getReg()); 1600 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1601 TII.get(TargetOpcode::COPY)) 1602 .addDef(ResVReg) 1603 .addUse(InputRegister) 1604 .constrainAllUses(TII, TRI, RBI); 1605 } 1606 1607 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1608 unsigned SpirvNotEqualId = 1609 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1610 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1611 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1612 Register NotEqualReg = ResVReg; 1613 1614 if (IsVectorTy) { 1615 NotEqualReg = IsBoolTy ? InputRegister 1616 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1617 const unsigned NumElts = InputType->getOperand(2).getImm(); 1618 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1619 } 1620 1621 if (!IsBoolTy) { 1622 Register ConstZeroReg = 1623 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1624 1625 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1626 .addDef(NotEqualReg) 1627 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1628 .addUse(InputRegister) 1629 .addUse(ConstZeroReg) 1630 .constrainAllUses(TII, TRI, RBI); 1631 } 1632 1633 if (!IsVectorTy) 1634 return true; 1635 1636 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1637 .addDef(ResVReg) 1638 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1639 .addUse(NotEqualReg) 1640 .constrainAllUses(TII, TRI, RBI); 1641 } 1642 1643 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1644 const SPIRVType *ResType, 1645 MachineInstr &I) const { 1646 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1647 } 1648 1649 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1650 const SPIRVType *ResType, 1651 MachineInstr &I) const { 1652 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1653 } 1654 1655 // Select the OpDot instruction for the given float dot 1656 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1657 const SPIRVType *ResType, 1658 MachineInstr &I) const { 1659 assert(I.getNumOperands() == 4); 1660 assert(I.getOperand(2).isReg()); 1661 assert(I.getOperand(3).isReg()); 1662 1663 [[maybe_unused]] SPIRVType *VecType = 1664 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1665 1666 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1667 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1668 "dot product requires a vector of at least 2 components"); 1669 1670 [[maybe_unused]] SPIRVType *EltType = 1671 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1672 1673 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1674 1675 MachineBasicBlock &BB = *I.getParent(); 1676 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1677 .addDef(ResVReg) 1678 .addUse(GR.getSPIRVTypeID(ResType)) 1679 .addUse(I.getOperand(2).getReg()) 1680 .addUse(I.getOperand(3).getReg()) 1681 .constrainAllUses(TII, TRI, RBI); 1682 } 1683 1684 // Since pre-1.6 SPIRV has no integer dot implementation, 1685 // expand by piecewise multiplying and adding the results 1686 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1687 const SPIRVType *ResType, 1688 MachineInstr &I) const { 1689 assert(I.getNumOperands() == 4); 1690 assert(I.getOperand(2).isReg()); 1691 assert(I.getOperand(3).isReg()); 1692 MachineBasicBlock &BB = *I.getParent(); 1693 1694 // Multiply the vectors, then sum the results 1695 Register Vec0 = I.getOperand(2).getReg(); 1696 Register Vec1 = I.getOperand(3).getReg(); 1697 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1698 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1699 1700 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1701 .addDef(TmpVec) 1702 .addUse(GR.getSPIRVTypeID(VecType)) 1703 .addUse(Vec0) 1704 .addUse(Vec1) 1705 .constrainAllUses(TII, TRI, RBI); 1706 1707 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1708 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1709 "dot product requires a vector of at least 2 components"); 1710 1711 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1712 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1713 .addDef(Res) 1714 .addUse(GR.getSPIRVTypeID(ResType)) 1715 .addUse(TmpVec) 1716 .addImm(0) 1717 .constrainAllUses(TII, TRI, RBI); 1718 1719 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1720 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1721 1722 Result &= 1723 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1724 .addDef(Elt) 1725 .addUse(GR.getSPIRVTypeID(ResType)) 1726 .addUse(TmpVec) 1727 .addImm(i) 1728 .constrainAllUses(TII, TRI, RBI); 1729 1730 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1731 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1732 : ResVReg; 1733 1734 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1735 .addDef(Sum) 1736 .addUse(GR.getSPIRVTypeID(ResType)) 1737 .addUse(Res) 1738 .addUse(Elt) 1739 .constrainAllUses(TII, TRI, RBI); 1740 Res = Sum; 1741 } 1742 1743 return Result; 1744 } 1745 1746 template <bool Signed> 1747 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1748 const SPIRVType *ResType, 1749 MachineInstr &I) const { 1750 assert(I.getNumOperands() == 5); 1751 assert(I.getOperand(2).isReg()); 1752 assert(I.getOperand(3).isReg()); 1753 assert(I.getOperand(4).isReg()); 1754 MachineBasicBlock &BB = *I.getParent(); 1755 1756 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1757 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1758 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1759 .addDef(Dot) 1760 .addUse(GR.getSPIRVTypeID(ResType)) 1761 .addUse(I.getOperand(2).getReg()) 1762 .addUse(I.getOperand(3).getReg()) 1763 .constrainAllUses(TII, TRI, RBI); 1764 1765 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1766 .addDef(ResVReg) 1767 .addUse(GR.getSPIRVTypeID(ResType)) 1768 .addUse(Dot) 1769 .addUse(I.getOperand(4).getReg()) 1770 .constrainAllUses(TII, TRI, RBI); 1771 1772 return Result; 1773 } 1774 1775 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1776 // extract the elements of the packed inputs, multiply them and add the result 1777 // to the accumulator. 1778 template <bool Signed> 1779 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1780 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1781 assert(I.getNumOperands() == 5); 1782 assert(I.getOperand(2).isReg()); 1783 assert(I.getOperand(3).isReg()); 1784 assert(I.getOperand(4).isReg()); 1785 MachineBasicBlock &BB = *I.getParent(); 1786 1787 bool Result = false; 1788 1789 // Acc = C 1790 Register Acc = I.getOperand(4).getReg(); 1791 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1792 auto ExtractOp = 1793 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1794 1795 // Extract the i8 element, multiply and add it to the accumulator 1796 for (unsigned i = 0; i < 4; i++) { 1797 // A[i] 1798 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1799 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1800 .addDef(AElt) 1801 .addUse(GR.getSPIRVTypeID(ResType)) 1802 .addUse(I.getOperand(2).getReg()) 1803 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1804 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1805 .constrainAllUses(TII, TRI, RBI); 1806 1807 // B[i] 1808 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1809 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1810 .addDef(BElt) 1811 .addUse(GR.getSPIRVTypeID(ResType)) 1812 .addUse(I.getOperand(3).getReg()) 1813 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1814 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1815 .constrainAllUses(TII, TRI, RBI); 1816 1817 // A[i] * B[i] 1818 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1819 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1820 .addDef(Mul) 1821 .addUse(GR.getSPIRVTypeID(ResType)) 1822 .addUse(AElt) 1823 .addUse(BElt) 1824 .constrainAllUses(TII, TRI, RBI); 1825 1826 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1827 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1828 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1829 .addDef(MaskMul) 1830 .addUse(GR.getSPIRVTypeID(ResType)) 1831 .addUse(Mul) 1832 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1833 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1834 .constrainAllUses(TII, TRI, RBI); 1835 1836 // Acc = Acc + A[i] * B[i] 1837 Register Sum = 1838 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1839 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1840 .addDef(Sum) 1841 .addUse(GR.getSPIRVTypeID(ResType)) 1842 .addUse(Acc) 1843 .addUse(MaskMul) 1844 .constrainAllUses(TII, TRI, RBI); 1845 1846 Acc = Sum; 1847 } 1848 1849 return Result; 1850 } 1851 1852 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1853 /// does not have a saturate builtin. 1854 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1855 const SPIRVType *ResType, 1856 MachineInstr &I) const { 1857 assert(I.getNumOperands() == 3); 1858 assert(I.getOperand(2).isReg()); 1859 MachineBasicBlock &BB = *I.getParent(); 1860 Register VZero = buildZerosValF(ResType, I); 1861 Register VOne = buildOnesValF(ResType, I); 1862 1863 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1864 .addDef(ResVReg) 1865 .addUse(GR.getSPIRVTypeID(ResType)) 1866 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1867 .addImm(GL::FClamp) 1868 .addUse(I.getOperand(2).getReg()) 1869 .addUse(VZero) 1870 .addUse(VOne) 1871 .constrainAllUses(TII, TRI, RBI); 1872 } 1873 1874 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1875 const SPIRVType *ResType, 1876 MachineInstr &I) const { 1877 assert(I.getNumOperands() == 3); 1878 assert(I.getOperand(2).isReg()); 1879 MachineBasicBlock &BB = *I.getParent(); 1880 Register InputRegister = I.getOperand(2).getReg(); 1881 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1882 auto &DL = I.getDebugLoc(); 1883 1884 if (!InputType) 1885 report_fatal_error("Input Type could not be determined."); 1886 1887 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1888 1889 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1890 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1891 1892 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1893 1894 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1895 Register SignReg = NeedsConversion 1896 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1897 : ResVReg; 1898 1899 bool Result = 1900 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1901 .addDef(SignReg) 1902 .addUse(GR.getSPIRVTypeID(InputType)) 1903 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1904 .addImm(SignOpcode) 1905 .addUse(InputRegister) 1906 .constrainAllUses(TII, TRI, RBI); 1907 1908 if (NeedsConversion) { 1909 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1910 Result |= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1911 .addDef(ResVReg) 1912 .addUse(GR.getSPIRVTypeID(ResType)) 1913 .addUse(SignReg) 1914 .constrainAllUses(TII, TRI, RBI); 1915 } 1916 1917 return Result; 1918 } 1919 1920 bool SPIRVInstructionSelector::selectWaveReadLaneAt(Register ResVReg, 1921 const SPIRVType *ResType, 1922 MachineInstr &I) const { 1923 assert(I.getNumOperands() == 4); 1924 assert(I.getOperand(2).isReg()); 1925 assert(I.getOperand(3).isReg()); 1926 MachineBasicBlock &BB = *I.getParent(); 1927 1928 // IntTy is used to define the execution scope, set to 3 to denote a 1929 // cross-lane interaction equivalent to a SPIR-V subgroup. 1930 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1931 return BuildMI(BB, I, I.getDebugLoc(), 1932 TII.get(SPIRV::OpGroupNonUniformShuffle)) 1933 .addDef(ResVReg) 1934 .addUse(GR.getSPIRVTypeID(ResType)) 1935 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)) 1936 .addUse(I.getOperand(2).getReg()) 1937 .addUse(I.getOperand(3).getReg()); 1938 } 1939 1940 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1941 const SPIRVType *ResType, 1942 MachineInstr &I) const { 1943 MachineBasicBlock &BB = *I.getParent(); 1944 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1945 .addDef(ResVReg) 1946 .addUse(GR.getSPIRVTypeID(ResType)) 1947 .addUse(I.getOperand(1).getReg()) 1948 .constrainAllUses(TII, TRI, RBI); 1949 } 1950 1951 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1952 const SPIRVType *ResType, 1953 MachineInstr &I) const { 1954 // There is no way to implement `freeze` correctly without support on SPIR-V 1955 // standard side, but we may at least address a simple (static) case when 1956 // undef/poison value presence is obvious. The main benefit of even 1957 // incomplete `freeze` support is preventing of translation from crashing due 1958 // to lack of support on legalization and instruction selection steps. 1959 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1960 return false; 1961 Register OpReg = I.getOperand(1).getReg(); 1962 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1963 Register Reg; 1964 switch (Def->getOpcode()) { 1965 case SPIRV::ASSIGN_TYPE: 1966 if (MachineInstr *AssignToDef = 1967 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1968 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1969 Reg = Def->getOperand(2).getReg(); 1970 } 1971 break; 1972 case SPIRV::OpUndef: 1973 Reg = Def->getOperand(1).getReg(); 1974 break; 1975 } 1976 unsigned DestOpCode; 1977 if (Reg.isValid()) { 1978 DestOpCode = SPIRV::OpConstantNull; 1979 } else { 1980 DestOpCode = TargetOpcode::COPY; 1981 Reg = OpReg; 1982 } 1983 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1984 .addDef(I.getOperand(0).getReg()) 1985 .addUse(Reg) 1986 .constrainAllUses(TII, TRI, RBI); 1987 } 1988 return false; 1989 } 1990 1991 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1992 const SPIRVType *ResType) { 1993 Register OpReg = ResType->getOperand(2).getReg(); 1994 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1995 if (!OpDef) 1996 return 0; 1997 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1998 OpDef->getOperand(1).isReg()) { 1999 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2000 OpDef = RefDef; 2001 } 2002 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2003 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2004 : 0; 2005 return N; 2006 } 2007 2008 // Return true if the type represents a constant register 2009 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2010 SmallPtrSet<SPIRVType *, 4> &Visited) { 2011 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2012 OpDef->getOperand(1).isReg()) { 2013 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2014 OpDef = RefDef; 2015 } 2016 2017 if (Visited.contains(OpDef)) 2018 return true; 2019 Visited.insert(OpDef); 2020 2021 unsigned Opcode = OpDef->getOpcode(); 2022 switch (Opcode) { 2023 case TargetOpcode::G_CONSTANT: 2024 case TargetOpcode::G_FCONSTANT: 2025 return true; 2026 case TargetOpcode::G_INTRINSIC: 2027 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2028 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2029 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2030 Intrinsic::spv_const_composite; 2031 case TargetOpcode::G_BUILD_VECTOR: 2032 case TargetOpcode::G_SPLAT_VECTOR: { 2033 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2034 i++) { 2035 SPIRVType *OpNestedDef = 2036 OpDef->getOperand(i).isReg() 2037 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2038 : nullptr; 2039 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2040 return false; 2041 } 2042 return true; 2043 } 2044 } 2045 return false; 2046 } 2047 2048 // Return true if the virtual register represents a constant 2049 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2050 SmallPtrSet<SPIRVType *, 4> Visited; 2051 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2052 return isConstReg(MRI, OpDef, Visited); 2053 return false; 2054 } 2055 2056 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2057 const SPIRVType *ResType, 2058 MachineInstr &I) const { 2059 unsigned N = 0; 2060 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2061 N = GR.getScalarOrVectorComponentCount(ResType); 2062 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2063 N = getArrayComponentCount(MRI, ResType); 2064 else 2065 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2066 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2067 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2068 2069 // check if we may construct a constant vector 2070 bool IsConst = true; 2071 for (unsigned i = I.getNumExplicitDefs(); 2072 i < I.getNumExplicitOperands() && IsConst; ++i) 2073 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2074 IsConst = false; 2075 2076 if (!IsConst && N < 2) 2077 report_fatal_error( 2078 "There must be at least two constituent operands in a vector"); 2079 2080 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2081 TII.get(IsConst ? SPIRV::OpConstantComposite 2082 : SPIRV::OpCompositeConstruct)) 2083 .addDef(ResVReg) 2084 .addUse(GR.getSPIRVTypeID(ResType)); 2085 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2086 MIB.addUse(I.getOperand(i).getReg()); 2087 return MIB.constrainAllUses(TII, TRI, RBI); 2088 } 2089 2090 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2091 const SPIRVType *ResType, 2092 MachineInstr &I) const { 2093 unsigned N = 0; 2094 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2095 N = GR.getScalarOrVectorComponentCount(ResType); 2096 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2097 N = getArrayComponentCount(MRI, ResType); 2098 else 2099 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2100 2101 unsigned OpIdx = I.getNumExplicitDefs(); 2102 if (!I.getOperand(OpIdx).isReg()) 2103 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2104 2105 // check if we may construct a constant vector 2106 Register OpReg = I.getOperand(OpIdx).getReg(); 2107 bool IsConst = isConstReg(MRI, OpReg); 2108 2109 if (!IsConst && N < 2) 2110 report_fatal_error( 2111 "There must be at least two constituent operands in a vector"); 2112 2113 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2114 TII.get(IsConst ? SPIRV::OpConstantComposite 2115 : SPIRV::OpCompositeConstruct)) 2116 .addDef(ResVReg) 2117 .addUse(GR.getSPIRVTypeID(ResType)); 2118 for (unsigned i = 0; i < N; ++i) 2119 MIB.addUse(OpReg); 2120 return MIB.constrainAllUses(TII, TRI, RBI); 2121 } 2122 2123 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2124 const SPIRVType *ResType, 2125 unsigned CmpOpc, 2126 MachineInstr &I) const { 2127 Register Cmp0 = I.getOperand(2).getReg(); 2128 Register Cmp1 = I.getOperand(3).getReg(); 2129 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2130 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2131 "CMP operands should have the same type"); 2132 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2133 .addDef(ResVReg) 2134 .addUse(GR.getSPIRVTypeID(ResType)) 2135 .addUse(Cmp0) 2136 .addUse(Cmp1) 2137 .constrainAllUses(TII, TRI, RBI); 2138 } 2139 2140 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2141 const SPIRVType *ResType, 2142 MachineInstr &I) const { 2143 auto Pred = I.getOperand(1).getPredicate(); 2144 unsigned CmpOpc; 2145 2146 Register CmpOperand = I.getOperand(2).getReg(); 2147 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2148 CmpOpc = getPtrCmpOpcode(Pred); 2149 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2150 CmpOpc = getBoolCmpOpcode(Pred); 2151 else 2152 CmpOpc = getICmpOpcode(Pred); 2153 return selectCmp(ResVReg, ResType, CmpOpc, I); 2154 } 2155 2156 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2157 const MachineInstr &I, 2158 int OpIdx) const { 2159 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2160 "Expected G_FCONSTANT"); 2161 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2162 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2163 } 2164 2165 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2166 const MachineInstr &I, 2167 int OpIdx) const { 2168 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2169 "Expected G_CONSTANT"); 2170 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2171 } 2172 2173 Register 2174 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2175 const SPIRVType *ResType) const { 2176 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2177 const SPIRVType *SpvI32Ty = 2178 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2179 // Find a constant in DT or build a new one. 2180 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2181 Register NewReg = GR.find(ConstInt, GR.CurMF); 2182 if (!NewReg.isValid()) { 2183 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2184 GR.add(ConstInt, GR.CurMF, NewReg); 2185 MachineInstr *MI; 2186 MachineBasicBlock &BB = *I.getParent(); 2187 if (Val == 0) { 2188 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2189 .addDef(NewReg) 2190 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2191 } else { 2192 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2193 .addDef(NewReg) 2194 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2195 .addImm(APInt(32, Val).getZExtValue()); 2196 } 2197 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2198 } 2199 return NewReg; 2200 } 2201 2202 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2203 const SPIRVType *ResType, 2204 MachineInstr &I) const { 2205 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2206 return selectCmp(ResVReg, ResType, CmpOp, I); 2207 } 2208 2209 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2210 MachineInstr &I) const { 2211 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2212 bool ZeroAsNull = STI.isOpenCLEnv(); 2213 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2214 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2215 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2216 } 2217 2218 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2219 MachineInstr &I) const { 2220 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2221 bool ZeroAsNull = STI.isOpenCLEnv(); 2222 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2223 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2224 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2225 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2226 } 2227 2228 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2229 MachineInstr &I) const { 2230 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2231 bool ZeroAsNull = STI.isOpenCLEnv(); 2232 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2233 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2234 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2235 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2236 } 2237 2238 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2239 const SPIRVType *ResType, 2240 MachineInstr &I) const { 2241 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2242 APInt One = 2243 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2244 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2245 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2246 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2247 } 2248 2249 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2250 const SPIRVType *ResType, 2251 MachineInstr &I, 2252 bool IsSigned) const { 2253 // To extend a bool, we need to use OpSelect between constants. 2254 Register ZeroReg = buildZerosVal(ResType, I); 2255 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2256 bool IsScalarBool = 2257 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2258 unsigned Opcode = 2259 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2260 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2261 .addDef(ResVReg) 2262 .addUse(GR.getSPIRVTypeID(ResType)) 2263 .addUse(I.getOperand(1).getReg()) 2264 .addUse(OneReg) 2265 .addUse(ZeroReg) 2266 .constrainAllUses(TII, TRI, RBI); 2267 } 2268 2269 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2270 const SPIRVType *ResType, 2271 MachineInstr &I, bool IsSigned, 2272 unsigned Opcode) const { 2273 Register SrcReg = I.getOperand(1).getReg(); 2274 // We can convert bool value directly to float type without OpConvert*ToF, 2275 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2276 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2277 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2278 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2279 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2280 const unsigned NumElts = ResType->getOperand(2).getImm(); 2281 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2282 } 2283 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2284 selectSelect(SrcReg, TmpType, I, false); 2285 } 2286 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2287 } 2288 2289 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2290 const SPIRVType *ResType, 2291 MachineInstr &I, bool IsSigned) const { 2292 Register SrcReg = I.getOperand(1).getReg(); 2293 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2294 return selectSelect(ResVReg, ResType, I, IsSigned); 2295 2296 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2297 if (SrcType == ResType) { 2298 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2299 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2300 if (DstRC != SrcRC && SrcRC) 2301 MRI->setRegClass(ResVReg, SrcRC); 2302 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2303 TII.get(TargetOpcode::COPY)) 2304 .addDef(ResVReg) 2305 .addUse(SrcReg) 2306 .constrainAllUses(TII, TRI, RBI); 2307 } 2308 2309 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2310 return selectUnOp(ResVReg, ResType, I, Opcode); 2311 } 2312 2313 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2314 Register ResVReg, 2315 MachineInstr &I, 2316 const SPIRVType *IntTy, 2317 const SPIRVType *BoolTy) const { 2318 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2319 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2320 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2321 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2322 Register Zero = buildZerosVal(IntTy, I); 2323 Register One = buildOnesVal(false, IntTy, I); 2324 MachineBasicBlock &BB = *I.getParent(); 2325 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2326 .addDef(BitIntReg) 2327 .addUse(GR.getSPIRVTypeID(IntTy)) 2328 .addUse(IntReg) 2329 .addUse(One) 2330 .constrainAllUses(TII, TRI, RBI); 2331 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2332 .addDef(ResVReg) 2333 .addUse(GR.getSPIRVTypeID(BoolTy)) 2334 .addUse(BitIntReg) 2335 .addUse(Zero) 2336 .constrainAllUses(TII, TRI, RBI); 2337 } 2338 2339 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2340 const SPIRVType *ResType, 2341 MachineInstr &I) const { 2342 Register IntReg = I.getOperand(1).getReg(); 2343 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2344 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2345 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2346 if (ArgType == ResType) { 2347 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2348 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2349 if (DstRC != SrcRC && SrcRC) 2350 MRI->setRegClass(ResVReg, SrcRC); 2351 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2352 TII.get(TargetOpcode::COPY)) 2353 .addDef(ResVReg) 2354 .addUse(IntReg) 2355 .constrainAllUses(TII, TRI, RBI); 2356 } 2357 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2358 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2359 return selectUnOp(ResVReg, ResType, I, Opcode); 2360 } 2361 2362 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2363 const SPIRVType *ResType, 2364 const APInt &Imm, 2365 MachineInstr &I) const { 2366 unsigned TyOpcode = ResType->getOpcode(); 2367 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2368 MachineBasicBlock &BB = *I.getParent(); 2369 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2370 Imm.isZero()) 2371 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2372 .addDef(ResVReg) 2373 .addUse(GR.getSPIRVTypeID(ResType)) 2374 .constrainAllUses(TII, TRI, RBI); 2375 if (TyOpcode == SPIRV::OpTypeInt) { 2376 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2377 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2378 if (Reg == ResVReg) 2379 return true; 2380 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2381 .addDef(ResVReg) 2382 .addUse(Reg) 2383 .constrainAllUses(TII, TRI, RBI); 2384 } 2385 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2386 .addDef(ResVReg) 2387 .addUse(GR.getSPIRVTypeID(ResType)); 2388 // <=32-bit integers should be caught by the sdag pattern. 2389 assert(Imm.getBitWidth() > 32); 2390 addNumImm(Imm, MIB); 2391 return MIB.constrainAllUses(TII, TRI, RBI); 2392 } 2393 2394 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2395 const SPIRVType *ResType, 2396 MachineInstr &I) const { 2397 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2398 .addDef(ResVReg) 2399 .addUse(GR.getSPIRVTypeID(ResType)) 2400 .constrainAllUses(TII, TRI, RBI); 2401 } 2402 2403 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2404 assert(MO.isReg()); 2405 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2406 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2407 assert(TypeInst->getOperand(1).isReg()); 2408 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2409 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2410 } 2411 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2412 } 2413 2414 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2415 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2416 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2417 return TypeInst->getOperand(2).getImm(); 2418 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2419 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2420 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2421 } 2422 2423 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2424 const SPIRVType *ResType, 2425 MachineInstr &I) const { 2426 MachineBasicBlock &BB = *I.getParent(); 2427 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2428 .addDef(ResVReg) 2429 .addUse(GR.getSPIRVTypeID(ResType)) 2430 // object to insert 2431 .addUse(I.getOperand(3).getReg()) 2432 // composite to insert into 2433 .addUse(I.getOperand(2).getReg()); 2434 for (unsigned i = 4; i < I.getNumOperands(); i++) 2435 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2436 return MIB.constrainAllUses(TII, TRI, RBI); 2437 } 2438 2439 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2440 const SPIRVType *ResType, 2441 MachineInstr &I) const { 2442 MachineBasicBlock &BB = *I.getParent(); 2443 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2444 .addDef(ResVReg) 2445 .addUse(GR.getSPIRVTypeID(ResType)) 2446 .addUse(I.getOperand(2).getReg()); 2447 for (unsigned i = 3; i < I.getNumOperands(); i++) 2448 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2449 return MIB.constrainAllUses(TII, TRI, RBI); 2450 } 2451 2452 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2453 const SPIRVType *ResType, 2454 MachineInstr &I) const { 2455 if (isImm(I.getOperand(4), MRI)) 2456 return selectInsertVal(ResVReg, ResType, I); 2457 MachineBasicBlock &BB = *I.getParent(); 2458 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2459 .addDef(ResVReg) 2460 .addUse(GR.getSPIRVTypeID(ResType)) 2461 .addUse(I.getOperand(2).getReg()) 2462 .addUse(I.getOperand(3).getReg()) 2463 .addUse(I.getOperand(4).getReg()) 2464 .constrainAllUses(TII, TRI, RBI); 2465 } 2466 2467 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2468 const SPIRVType *ResType, 2469 MachineInstr &I) const { 2470 if (isImm(I.getOperand(3), MRI)) 2471 return selectExtractVal(ResVReg, ResType, I); 2472 MachineBasicBlock &BB = *I.getParent(); 2473 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2474 .addDef(ResVReg) 2475 .addUse(GR.getSPIRVTypeID(ResType)) 2476 .addUse(I.getOperand(2).getReg()) 2477 .addUse(I.getOperand(3).getReg()) 2478 .constrainAllUses(TII, TRI, RBI); 2479 } 2480 2481 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2482 const SPIRVType *ResType, 2483 MachineInstr &I) const { 2484 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2485 2486 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2487 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2488 // we have to use Op[InBounds]AccessChain. 2489 const unsigned Opcode = STI.isVulkanEnv() 2490 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2491 : SPIRV::OpAccessChain) 2492 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2493 : SPIRV::OpPtrAccessChain); 2494 2495 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2496 .addDef(ResVReg) 2497 .addUse(GR.getSPIRVTypeID(ResType)) 2498 // Object to get a pointer to. 2499 .addUse(I.getOperand(3).getReg()); 2500 // Adding indices. 2501 const unsigned StartingIndex = 2502 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2503 ? 5 2504 : 4; 2505 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2506 Res.addUse(I.getOperand(i).getReg()); 2507 return Res.constrainAllUses(TII, TRI, RBI); 2508 } 2509 2510 // Maybe wrap a value into OpSpecConstantOp 2511 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2512 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2513 bool Result = true; 2514 unsigned Lim = I.getNumExplicitOperands(); 2515 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2516 Register OpReg = I.getOperand(i).getReg(); 2517 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2518 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2519 SmallPtrSet<SPIRVType *, 4> Visited; 2520 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2521 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2522 GR.isAggregateType(OpType)) { 2523 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2524 // by selectAddrSpaceCast() 2525 CompositeArgs.push_back(OpReg); 2526 continue; 2527 } 2528 MachineFunction *MF = I.getMF(); 2529 Register WrapReg = GR.find(OpDefine, MF); 2530 if (WrapReg.isValid()) { 2531 CompositeArgs.push_back(WrapReg); 2532 continue; 2533 } 2534 // Create a new register for the wrapper 2535 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2536 GR.add(OpDefine, MF, WrapReg); 2537 CompositeArgs.push_back(WrapReg); 2538 // Decorate the wrapper register and generate a new instruction 2539 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2540 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2541 MachineBasicBlock &BB = *I.getParent(); 2542 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2543 .addDef(WrapReg) 2544 .addUse(GR.getSPIRVTypeID(OpType)) 2545 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2546 .addUse(OpReg) 2547 .constrainAllUses(TII, TRI, RBI); 2548 if (!Result) 2549 break; 2550 } 2551 return Result; 2552 } 2553 2554 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2555 const SPIRVType *ResType, 2556 MachineInstr &I) const { 2557 MachineBasicBlock &BB = *I.getParent(); 2558 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2559 switch (IID) { 2560 case Intrinsic::spv_load: 2561 return selectLoad(ResVReg, ResType, I); 2562 case Intrinsic::spv_store: 2563 return selectStore(I); 2564 case Intrinsic::spv_extractv: 2565 return selectExtractVal(ResVReg, ResType, I); 2566 case Intrinsic::spv_insertv: 2567 return selectInsertVal(ResVReg, ResType, I); 2568 case Intrinsic::spv_extractelt: 2569 return selectExtractElt(ResVReg, ResType, I); 2570 case Intrinsic::spv_insertelt: 2571 return selectInsertElt(ResVReg, ResType, I); 2572 case Intrinsic::spv_gep: 2573 return selectGEP(ResVReg, ResType, I); 2574 case Intrinsic::spv_unref_global: 2575 case Intrinsic::spv_init_global: { 2576 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2577 MachineInstr *Init = I.getNumExplicitOperands() > 2 2578 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2579 : nullptr; 2580 assert(MI); 2581 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2582 } 2583 case Intrinsic::spv_undef: { 2584 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2585 .addDef(ResVReg) 2586 .addUse(GR.getSPIRVTypeID(ResType)); 2587 return MIB.constrainAllUses(TII, TRI, RBI); 2588 } 2589 case Intrinsic::spv_const_composite: { 2590 // If no values are attached, the composite is null constant. 2591 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2592 // Select a proper instruction. 2593 unsigned Opcode = SPIRV::OpConstantNull; 2594 SmallVector<Register> CompositeArgs; 2595 if (!IsNull) { 2596 Opcode = SPIRV::OpConstantComposite; 2597 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2598 return false; 2599 } 2600 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2601 .addDef(ResVReg) 2602 .addUse(GR.getSPIRVTypeID(ResType)); 2603 // skip type MD node we already used when generated assign.type for this 2604 if (!IsNull) { 2605 for (Register OpReg : CompositeArgs) 2606 MIB.addUse(OpReg); 2607 } 2608 return MIB.constrainAllUses(TII, TRI, RBI); 2609 } 2610 case Intrinsic::spv_assign_name: { 2611 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2612 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2613 for (unsigned i = I.getNumExplicitDefs() + 2; 2614 i < I.getNumExplicitOperands(); ++i) { 2615 MIB.addImm(I.getOperand(i).getImm()); 2616 } 2617 return MIB.constrainAllUses(TII, TRI, RBI); 2618 } 2619 case Intrinsic::spv_switch: { 2620 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2621 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2622 if (I.getOperand(i).isReg()) 2623 MIB.addReg(I.getOperand(i).getReg()); 2624 else if (I.getOperand(i).isCImm()) 2625 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2626 else if (I.getOperand(i).isMBB()) 2627 MIB.addMBB(I.getOperand(i).getMBB()); 2628 else 2629 llvm_unreachable("Unexpected OpSwitch operand"); 2630 } 2631 return MIB.constrainAllUses(TII, TRI, RBI); 2632 } 2633 case Intrinsic::spv_loop_merge: 2634 case Intrinsic::spv_selection_merge: { 2635 const auto Opcode = IID == Intrinsic::spv_selection_merge 2636 ? SPIRV::OpSelectionMerge 2637 : SPIRV::OpLoopMerge; 2638 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2639 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2640 assert(I.getOperand(i).isMBB()); 2641 MIB.addMBB(I.getOperand(i).getMBB()); 2642 } 2643 MIB.addImm(SPIRV::SelectionControl::None); 2644 return MIB.constrainAllUses(TII, TRI, RBI); 2645 } 2646 case Intrinsic::spv_cmpxchg: 2647 return selectAtomicCmpXchg(ResVReg, ResType, I); 2648 case Intrinsic::spv_unreachable: 2649 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 2650 break; 2651 case Intrinsic::spv_alloca: 2652 return selectFrameIndex(ResVReg, ResType, I); 2653 case Intrinsic::spv_alloca_array: 2654 return selectAllocaArray(ResVReg, ResType, I); 2655 case Intrinsic::spv_assume: 2656 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2657 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2658 .addUse(I.getOperand(1).getReg()); 2659 break; 2660 case Intrinsic::spv_expect: 2661 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2662 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2663 .addDef(ResVReg) 2664 .addUse(GR.getSPIRVTypeID(ResType)) 2665 .addUse(I.getOperand(2).getReg()) 2666 .addUse(I.getOperand(3).getReg()); 2667 break; 2668 case Intrinsic::arithmetic_fence: 2669 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2670 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT)) 2671 .addDef(ResVReg) 2672 .addUse(GR.getSPIRVTypeID(ResType)) 2673 .addUse(I.getOperand(2).getReg()); 2674 else 2675 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 2676 .addUse(I.getOperand(2).getReg()); 2677 break; 2678 case Intrinsic::spv_thread_id: 2679 return selectSpvThreadId(ResVReg, ResType, I); 2680 case Intrinsic::spv_fdot: 2681 return selectFloatDot(ResVReg, ResType, I); 2682 case Intrinsic::spv_udot: 2683 case Intrinsic::spv_sdot: 2684 return selectIntegerDot(ResVReg, ResType, I); 2685 case Intrinsic::spv_dot4add_i8packed: 2686 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2687 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2688 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2689 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2690 case Intrinsic::spv_dot4add_u8packed: 2691 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2692 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2693 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2694 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2695 case Intrinsic::spv_all: 2696 return selectAll(ResVReg, ResType, I); 2697 case Intrinsic::spv_any: 2698 return selectAny(ResVReg, ResType, I); 2699 case Intrinsic::spv_cross: 2700 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2701 case Intrinsic::spv_lerp: 2702 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2703 case Intrinsic::spv_length: 2704 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2705 case Intrinsic::spv_degrees: 2706 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2707 case Intrinsic::spv_frac: 2708 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2709 case Intrinsic::spv_normalize: 2710 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2711 case Intrinsic::spv_rsqrt: 2712 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2713 case Intrinsic::spv_sign: 2714 return selectSign(ResVReg, ResType, I); 2715 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2716 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2717 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2718 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2719 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2720 Register MemSemReg = 2721 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2722 Register ScopeReg = buildI32Constant(SPIRV::Scope::Workgroup, I); 2723 MachineBasicBlock &BB = *I.getParent(); 2724 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2725 .addUse(ScopeReg) 2726 .addUse(ScopeReg) 2727 .addUse(MemSemReg) 2728 .constrainAllUses(TII, TRI, RBI); 2729 } break; 2730 case Intrinsic::spv_lifetime_start: 2731 case Intrinsic::spv_lifetime_end: { 2732 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2733 : SPIRV::OpLifetimeStop; 2734 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2735 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2736 if (Size == -1) 2737 Size = 0; 2738 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 2739 } break; 2740 case Intrinsic::spv_saturate: 2741 return selectSaturate(ResVReg, ResType, I); 2742 case Intrinsic::spv_wave_is_first_lane: { 2743 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2744 return BuildMI(BB, I, I.getDebugLoc(), 2745 TII.get(SPIRV::OpGroupNonUniformElect)) 2746 .addDef(ResVReg) 2747 .addUse(GR.getSPIRVTypeID(ResType)) 2748 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2749 } 2750 case Intrinsic::spv_wave_readlane: 2751 return selectWaveReadLaneAt(ResVReg, ResType, I); 2752 case Intrinsic::spv_step: 2753 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2754 case Intrinsic::spv_radians: 2755 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2756 // Discard intrinsics which we do not expect to actually represent code after 2757 // lowering or intrinsics which are not implemented but should not crash when 2758 // found in a customer's LLVM IR input. 2759 case Intrinsic::instrprof_increment: 2760 case Intrinsic::instrprof_increment_step: 2761 case Intrinsic::instrprof_value_profile: 2762 break; 2763 // Discard internal intrinsics. 2764 case Intrinsic::spv_value_md: 2765 break; 2766 case Intrinsic::spv_handle_fromBinding: { 2767 selectHandleFromBinding(ResVReg, ResType, I); 2768 return true; 2769 } 2770 default: { 2771 std::string DiagMsg; 2772 raw_string_ostream OS(DiagMsg); 2773 I.print(OS); 2774 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2775 report_fatal_error(DiagMsg.c_str(), false); 2776 } 2777 } 2778 return true; 2779 } 2780 2781 void SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2782 const SPIRVType *ResType, 2783 MachineInstr &I) const { 2784 2785 uint32_t Set = foldImm(I.getOperand(2), MRI); 2786 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2787 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2788 Register IndexReg = I.getOperand(5).getReg(); 2789 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2790 2791 MachineIRBuilder MIRBuilder(I); 2792 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2793 IndexReg, IsNonUniform, MIRBuilder); 2794 2795 if (IsNonUniform) 2796 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2797 2798 // TODO: For now we assume the resource is an image, which needs to be 2799 // loaded to get the handle. That will not be true for storage buffers. 2800 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2801 .addDef(ResVReg) 2802 .addUse(GR.getSPIRVTypeID(ResType)) 2803 .addUse(VarReg); 2804 } 2805 2806 Register SPIRVInstructionSelector::buildPointerToResource( 2807 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 2808 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 2809 MachineIRBuilder MIRBuilder) const { 2810 if (ArraySize == 1) 2811 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 2812 MIRBuilder); 2813 2814 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 2815 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 2816 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 2817 VarType, Set, Binding, MIRBuilder); 2818 2819 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 2820 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 2821 2822 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2823 if (IsNonUniform) { 2824 // It is unclear which value needs to be marked an non-uniform, so both 2825 // the index and the access changed are decorated as non-uniform. 2826 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2827 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2828 } 2829 2830 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 2831 .addDef(AcReg) 2832 .addUse(GR.getSPIRVTypeID(ResPointerType)) 2833 .addUse(VarReg) 2834 .addUse(IndexReg); 2835 2836 return AcReg; 2837 } 2838 2839 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 2840 const SPIRVType *ResType, 2841 MachineInstr &I, 2842 bool IsSigned) const { 2843 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2844 // zero or sign extend 2845 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2846 bool Result = 2847 selectUnOpWithSrc(ExtReg, ResType, I, I.getOperand(2).getReg(), Opcode); 2848 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 2849 } 2850 2851 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 2852 const SPIRVType *ResType, 2853 MachineInstr &I, 2854 Register SrcReg, 2855 bool IsSigned) const { 2856 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 2857 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2858 .addDef(ResVReg) 2859 .addUse(GR.getSPIRVTypeID(ResType)) 2860 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2861 .addImm(Opcode) 2862 .addUse(SrcReg) 2863 .constrainAllUses(TII, TRI, RBI); 2864 } 2865 2866 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 2867 const SPIRVType *ResType, 2868 MachineInstr &I, 2869 bool IsSigned) const { 2870 Register OpReg = I.getOperand(2).getReg(); 2871 // 1. split our int64 into 2 pieces using a bitcast 2872 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 2873 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 2874 MachineIRBuilder MIRBuilder(I); 2875 SPIRVType *postCastT = 2876 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 2877 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 2878 bool Result = 2879 selectUnOpWithSrc(bitcastReg, postCastT, I, OpReg, SPIRV::OpBitcast); 2880 2881 // 2. call firstbithigh 2882 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 2883 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 2884 2885 // 3. split result vector into high bits and low bits 2886 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2887 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2888 2889 bool ZeroAsNull = STI.isOpenCLEnv(); 2890 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 2891 if (isScalarRes) { 2892 // if scalar do a vector extract 2893 Result &= selectNAryOpWithSrcs( 2894 HighReg, ResType, I, 2895 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 2896 SPIRV::OpVectorExtractDynamic); 2897 Result &= selectNAryOpWithSrcs( 2898 LowReg, ResType, I, 2899 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 2900 SPIRV::OpVectorExtractDynamic); 2901 } else { // vector case do a shufflevector 2902 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2903 TII.get(SPIRV::OpVectorShuffle)) 2904 .addDef(HighReg) 2905 .addUse(GR.getSPIRVTypeID(ResType)) 2906 .addUse(FBHReg) 2907 .addUse(FBHReg); 2908 // ^^ this vector will not be selected from; could be empty 2909 unsigned j; 2910 for (j = 0; j < count * 2; j += 2) { 2911 MIB.addImm(j); 2912 } 2913 Result &= MIB.constrainAllUses(TII, TRI, RBI); 2914 2915 // get low bits 2916 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2917 TII.get(SPIRV::OpVectorShuffle)) 2918 .addDef(LowReg) 2919 .addUse(GR.getSPIRVTypeID(ResType)) 2920 .addUse(FBHReg) 2921 .addUse(FBHReg); 2922 // ^^ this vector will not be selected from; could be empty 2923 for (j = 1; j < count * 2; j += 2) { 2924 MIB.addImm(j); 2925 } 2926 Result &= MIB.constrainAllUses(TII, TRI, RBI); 2927 } 2928 2929 // 4. check if result of each top 32 bits is == -1 2930 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2931 Register NegOneReg; 2932 Register Reg0; 2933 Register Reg32; 2934 unsigned selectOp; 2935 unsigned addOp; 2936 if (isScalarRes) { 2937 NegOneReg = 2938 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 2939 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2940 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 2941 selectOp = SPIRV::OpSelectSISCond; 2942 addOp = SPIRV::OpIAddS; 2943 } else { 2944 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 2945 NegOneReg = 2946 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 2947 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 2948 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 2949 selectOp = SPIRV::OpSelectVIVCond; 2950 addOp = SPIRV::OpIAddV; 2951 } 2952 2953 // check if the high bits are == -1; true if -1 2954 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 2955 Result &= selectNAryOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 2956 SPIRV::OpIEqual); 2957 2958 // Select low bits if true in BReg, otherwise high bits 2959 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2960 Result &= selectNAryOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, 2961 selectOp); 2962 2963 // Add 32 for high bits, 0 for low bits 2964 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2965 Result &= 2966 selectNAryOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 2967 2968 return Result && 2969 selectNAryOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 2970 } 2971 2972 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 2973 const SPIRVType *ResType, 2974 MachineInstr &I, 2975 bool IsSigned) const { 2976 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 2977 Register OpReg = I.getOperand(2).getReg(); 2978 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2979 2980 switch (GR.getScalarOrVectorBitWidth(OpType)) { 2981 case 16: 2982 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 2983 case 32: 2984 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 2985 case 64: 2986 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 2987 default: 2988 report_fatal_error( 2989 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 2990 } 2991 } 2992 2993 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 2994 const SPIRVType *ResType, 2995 MachineInstr &I) const { 2996 // there was an allocation size parameter to the allocation instruction 2997 // that is not 1 2998 MachineBasicBlock &BB = *I.getParent(); 2999 return BuildMI(BB, I, I.getDebugLoc(), 3000 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3001 .addDef(ResVReg) 3002 .addUse(GR.getSPIRVTypeID(ResType)) 3003 .addUse(I.getOperand(2).getReg()) 3004 .constrainAllUses(TII, TRI, RBI); 3005 } 3006 3007 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3008 const SPIRVType *ResType, 3009 MachineInstr &I) const { 3010 // Change order of instructions if needed: all OpVariable instructions in a 3011 // function must be the first instructions in the first block 3012 MachineFunction *MF = I.getParent()->getParent(); 3013 MachineBasicBlock *MBB = &MF->front(); 3014 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 3015 bool IsHeader = false; 3016 unsigned Opcode; 3017 for (; It != E && It != I; ++It) { 3018 Opcode = It->getOpcode(); 3019 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 3020 IsHeader = true; 3021 } else if (IsHeader && 3022 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 3023 ++It; 3024 break; 3025 } 3026 } 3027 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 3028 .addDef(ResVReg) 3029 .addUse(GR.getSPIRVTypeID(ResType)) 3030 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3031 .constrainAllUses(TII, TRI, RBI); 3032 } 3033 3034 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3035 // InstructionSelector walks backwards through the instructions. We can use 3036 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3037 // first, so can generate an OpBranchConditional here. If there is no 3038 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3039 const MachineInstr *PrevI = I.getPrevNode(); 3040 MachineBasicBlock &MBB = *I.getParent(); 3041 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3042 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3043 .addUse(PrevI->getOperand(0).getReg()) 3044 .addMBB(PrevI->getOperand(1).getMBB()) 3045 .addMBB(I.getOperand(0).getMBB()) 3046 .constrainAllUses(TII, TRI, RBI); 3047 } 3048 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3049 .addMBB(I.getOperand(0).getMBB()) 3050 .constrainAllUses(TII, TRI, RBI); 3051 } 3052 3053 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3054 // InstructionSelector walks backwards through the instructions. For an 3055 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3056 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3057 // generate the OpBranchConditional in selectBranch above. 3058 // 3059 // If an OpBranchConditional has been generated, we simply return, as the work 3060 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3061 // implicit fallthrough to the next basic block, so we need to create an 3062 // OpBranchConditional with an explicit "false" argument pointing to the next 3063 // basic block that LLVM would fall through to. 3064 const MachineInstr *NextI = I.getNextNode(); 3065 // Check if this has already been successfully selected. 3066 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3067 return true; 3068 // Must be relying on implicit block fallthrough, so generate an 3069 // OpBranchConditional with the "next" basic block as the "false" target. 3070 MachineBasicBlock &MBB = *I.getParent(); 3071 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3072 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3073 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3074 .addUse(I.getOperand(0).getReg()) 3075 .addMBB(I.getOperand(1).getMBB()) 3076 .addMBB(NextMBB) 3077 .constrainAllUses(TII, TRI, RBI); 3078 } 3079 3080 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3081 const SPIRVType *ResType, 3082 MachineInstr &I) const { 3083 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3084 .addDef(ResVReg) 3085 .addUse(GR.getSPIRVTypeID(ResType)); 3086 const unsigned NumOps = I.getNumOperands(); 3087 for (unsigned i = 1; i < NumOps; i += 2) { 3088 MIB.addUse(I.getOperand(i + 0).getReg()); 3089 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3090 } 3091 return MIB.constrainAllUses(TII, TRI, RBI); 3092 } 3093 3094 bool SPIRVInstructionSelector::selectGlobalValue( 3095 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3096 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3097 MachineIRBuilder MIRBuilder(I); 3098 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3099 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3100 SPIRVType *PointerBaseType; 3101 if (GVType->isArrayTy()) { 3102 SPIRVType *ArrayElementType = 3103 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3104 SPIRV::AccessQualifier::ReadWrite, false); 3105 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3106 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3107 } else { 3108 PointerBaseType = GR.getOrCreateSPIRVType( 3109 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3110 } 3111 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3112 PointerBaseType, I, TII, 3113 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3114 3115 std::string GlobalIdent; 3116 if (!GV->hasName()) { 3117 unsigned &ID = UnnamedGlobalIDs[GV]; 3118 if (ID == 0) 3119 ID = UnnamedGlobalIDs.size(); 3120 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3121 } else { 3122 GlobalIdent = GV->getGlobalIdentifier(); 3123 } 3124 3125 // Behaviour of functions as operands depends on availability of the 3126 // corresponding extension (SPV_INTEL_function_pointers): 3127 // - If there is an extension to operate with functions as operands: 3128 // We create a proper constant operand and evaluate a correct type for a 3129 // function pointer. 3130 // - Without the required extension: 3131 // We have functions as operands in tests with blocks of instruction e.g. in 3132 // transcoding/global_block.ll. These operands are not used and should be 3133 // substituted by zero constants. Their type is expected to be always 3134 // OpTypePointer Function %uchar. 3135 if (isa<Function>(GV)) { 3136 const Constant *ConstVal = GV; 3137 MachineBasicBlock &BB = *I.getParent(); 3138 Register NewReg = GR.find(ConstVal, GR.CurMF); 3139 if (!NewReg.isValid()) { 3140 Register NewReg = ResVReg; 3141 GR.add(ConstVal, GR.CurMF, NewReg); 3142 const Function *GVFun = 3143 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3144 ? dyn_cast<Function>(GV) 3145 : nullptr; 3146 if (GVFun) { 3147 // References to a function via function pointers generate virtual 3148 // registers without a definition. We will resolve it later, during 3149 // module analysis stage. 3150 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3151 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3152 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3153 MachineInstrBuilder MB = 3154 BuildMI(BB, I, I.getDebugLoc(), 3155 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3156 .addDef(NewReg) 3157 .addUse(GR.getSPIRVTypeID(ResType)) 3158 .addUse(FuncVReg); 3159 // mapping the function pointer to the used Function 3160 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3161 return MB.constrainAllUses(TII, TRI, RBI); 3162 } 3163 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3164 .addDef(NewReg) 3165 .addUse(GR.getSPIRVTypeID(ResType)) 3166 .constrainAllUses(TII, TRI, RBI); 3167 } 3168 assert(NewReg != ResVReg); 3169 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 3170 .addDef(ResVReg) 3171 .addUse(NewReg) 3172 .constrainAllUses(TII, TRI, RBI); 3173 } 3174 auto GlobalVar = cast<GlobalVariable>(GV); 3175 assert(GlobalVar->getName() != "llvm.global.annotations"); 3176 3177 bool HasInit = GlobalVar->hasInitializer() && 3178 !isa<UndefValue>(GlobalVar->getInitializer()); 3179 // Skip empty declaration for GVs with initilaizers till we get the decl with 3180 // passed initializer. 3181 if (HasInit && !Init) 3182 return true; 3183 3184 unsigned AddrSpace = GV->getAddressSpace(); 3185 SPIRV::StorageClass::StorageClass Storage = 3186 addressSpaceToStorageClass(AddrSpace, STI); 3187 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 3188 Storage != SPIRV::StorageClass::Function; 3189 SPIRV::LinkageType::LinkageType LnkType = 3190 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3191 ? SPIRV::LinkageType::Import 3192 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3193 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3194 ? SPIRV::LinkageType::LinkOnceODR 3195 : SPIRV::LinkageType::Export); 3196 3197 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 3198 Storage, Init, GlobalVar->isConstant(), 3199 HasLnkTy, LnkType, MIRBuilder, true); 3200 return Reg.isValid(); 3201 } 3202 3203 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3204 const SPIRVType *ResType, 3205 MachineInstr &I) const { 3206 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3207 return selectExtInst(ResVReg, ResType, I, CL::log10); 3208 } 3209 3210 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3211 // is implemented as: 3212 // log10(x) = log2(x) * (1 / log2(10)) 3213 // = log2(x) * 0.30103 3214 3215 MachineIRBuilder MIRBuilder(I); 3216 MachineBasicBlock &BB = *I.getParent(); 3217 3218 // Build log2(x). 3219 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3220 bool Result = 3221 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3222 .addDef(VarReg) 3223 .addUse(GR.getSPIRVTypeID(ResType)) 3224 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3225 .addImm(GL::Log2) 3226 .add(I.getOperand(1)) 3227 .constrainAllUses(TII, TRI, RBI); 3228 3229 // Build 0.30103. 3230 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3231 ResType->getOpcode() == SPIRV::OpTypeFloat); 3232 // TODO: Add matrix implementation once supported by the HLSL frontend. 3233 const SPIRVType *SpirvScalarType = 3234 ResType->getOpcode() == SPIRV::OpTypeVector 3235 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3236 : ResType; 3237 Register ScaleReg = 3238 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3239 3240 // Multiply log2(x) by 0.30103 to get log10(x) result. 3241 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3242 ? SPIRV::OpVectorTimesScalar 3243 : SPIRV::OpFMulS; 3244 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3245 .addDef(ResVReg) 3246 .addUse(GR.getSPIRVTypeID(ResType)) 3247 .addUse(VarReg) 3248 .addUse(ScaleReg) 3249 .constrainAllUses(TII, TRI, RBI); 3250 3251 return Result; 3252 } 3253 3254 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3255 const SPIRVType *ResType, 3256 MachineInstr &I) const { 3257 // DX intrinsic: @llvm.dx.thread.id(i32) 3258 // ID Name Description 3259 // 93 ThreadId reads the thread ID 3260 3261 MachineIRBuilder MIRBuilder(I); 3262 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3263 const SPIRVType *Vec3Ty = 3264 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3265 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3266 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3267 3268 // Create new register for GlobalInvocationID builtin variable. 3269 Register NewRegister = 3270 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3271 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3272 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3273 3274 // Build GlobalInvocationID global variable with the necessary decorations. 3275 Register Variable = GR.buildGlobalVariable( 3276 NewRegister, PtrType, 3277 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3278 SPIRV::StorageClass::Input, nullptr, true, true, 3279 SPIRV::LinkageType::Import, MIRBuilder, false); 3280 3281 // Create new register for loading value. 3282 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3283 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3284 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3285 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3286 3287 // Load v3uint value from the global variable. 3288 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3289 .addDef(LoadedRegister) 3290 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3291 .addUse(Variable); 3292 3293 // Get Thread ID index. Expecting operand is a constant immediate value, 3294 // wrapped in a type assignment. 3295 assert(I.getOperand(2).isReg()); 3296 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3297 3298 // Extract the thread ID from the loaded vector value. 3299 MachineBasicBlock &BB = *I.getParent(); 3300 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3301 .addDef(ResVReg) 3302 .addUse(GR.getSPIRVTypeID(ResType)) 3303 .addUse(LoadedRegister) 3304 .addImm(ThreadId); 3305 return MIB.constrainAllUses(TII, TRI, RBI); 3306 } 3307 3308 namespace llvm { 3309 InstructionSelector * 3310 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3311 const SPIRVSubtarget &Subtarget, 3312 const RegisterBankInfo &RBI) { 3313 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3314 } 3315 } // namespace llvm 3316