1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 37 #define DEBUG_TYPE "spirv-isel" 38 39 using namespace llvm; 40 namespace CL = SPIRV::OpenCLExtInst; 41 namespace GL = SPIRV::GLSLExtInst; 42 43 using ExtInstList = 44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 45 46 namespace { 47 48 #define GET_GLOBALISEL_PREDICATE_BITSET 49 #include "SPIRVGenGlobalISel.inc" 50 #undef GET_GLOBALISEL_PREDICATE_BITSET 51 52 class SPIRVInstructionSelector : public InstructionSelector { 53 const SPIRVSubtarget &STI; 54 const SPIRVInstrInfo &TII; 55 const SPIRVRegisterInfo &TRI; 56 const RegisterBankInfo &RBI; 57 SPIRVGlobalRegistry &GR; 58 MachineRegisterInfo *MRI; 59 MachineFunction *HasVRegsReset = nullptr; 60 61 /// We need to keep track of the number we give to anonymous global values to 62 /// generate the same name every time when this is needed. 63 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 64 65 public: 66 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 67 const SPIRVSubtarget &ST, 68 const RegisterBankInfo &RBI); 69 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 70 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 71 BlockFrequencyInfo *BFI) override; 72 // Common selection code. Instruction-specific selection occurs in spvSelect. 73 bool select(MachineInstr &I) override; 74 static const char *getName() { return DEBUG_TYPE; } 75 76 #define GET_GLOBALISEL_PREDICATES_DECL 77 #include "SPIRVGenGlobalISel.inc" 78 #undef GET_GLOBALISEL_PREDICATES_DECL 79 80 #define GET_GLOBALISEL_TEMPORARIES_DECL 81 #include "SPIRVGenGlobalISel.inc" 82 #undef GET_GLOBALISEL_TEMPORARIES_DECL 83 84 private: 85 void resetVRegsType(MachineFunction &MF); 86 87 // tblgen-erated 'select' implementation, used as the initial selector for 88 // the patterns that don't require complex C++. 89 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 90 91 // All instruction-specific selection that didn't happen in "select()". 92 // Is basically a large Switch/Case delegating to all other select method. 93 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 94 MachineInstr &I) const; 95 96 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 97 MachineInstr &I, bool IsSigned) const; 98 99 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 100 MachineInstr &I, bool IsSigned) const; 101 102 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 103 MachineInstr &I, Register SrcReg, 104 bool IsSigned) const; 105 106 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I, bool IsSigned) const; 108 109 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 110 const MachineInstr *Init = nullptr) const; 111 112 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I, std::vector<Register> SrcRegs, 114 unsigned Opcode) const; 115 116 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 117 unsigned Opcode) const; 118 119 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I) const; 121 122 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 123 MachineInstr &I) const; 124 bool selectStore(MachineInstr &I) const; 125 126 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 bool selectStackRestore(MachineInstr &I) const; 129 130 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 131 132 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I, unsigned NewOpcode, 134 unsigned NegateOpcode = 0) const; 135 136 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 137 MachineInstr &I) const; 138 139 bool selectFence(MachineInstr &I) const; 140 141 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 142 MachineInstr &I) const; 143 144 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 145 MachineInstr &I, unsigned OpType) const; 146 147 bool selectAll(Register ResVReg, const SPIRVType *ResType, 148 MachineInstr &I) const; 149 150 bool selectAny(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 153 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 154 MachineInstr &I) const; 155 156 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 157 MachineInstr &I) const; 158 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 162 unsigned comparisonOpcode, MachineInstr &I) const; 163 bool selectCross(Register ResVReg, const SPIRVType *ResType, 164 MachineInstr &I) const; 165 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 168 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectSign(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I, unsigned Opcode) const; 181 182 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I) const; 184 185 template <bool Signed> 186 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 template <bool Signed> 189 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 190 MachineInstr &I) const; 191 192 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 193 int OpIdx) const; 194 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 195 int OpIdx) const; 196 197 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 198 MachineInstr &I) const; 199 200 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 201 bool IsSigned) const; 202 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 203 bool IsSigned, unsigned Opcode) const; 204 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 205 bool IsSigned) const; 206 207 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 208 MachineInstr &I) const; 209 210 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 211 const SPIRVType *intTy, const SPIRVType *boolTy) const; 212 213 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 214 MachineInstr &I) const; 215 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 216 MachineInstr &I) const; 217 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 218 MachineInstr &I) const; 219 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 220 MachineInstr &I) const; 221 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I) const; 223 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 224 MachineInstr &I) const; 225 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 226 MachineInstr &I) const; 227 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I) const; 229 230 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 233 MachineInstr &I) const; 234 235 bool selectBranch(MachineInstr &I) const; 236 bool selectBranchCond(MachineInstr &I) const; 237 238 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 239 MachineInstr &I) const; 240 241 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 242 MachineInstr &I, GL::GLSLExtInst GLInst) const; 243 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 244 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 245 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 246 MachineInstr &I, CL::OpenCLExtInst CLInst, 247 GL::GLSLExtInst GLInst) const; 248 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 249 MachineInstr &I, const ExtInstList &ExtInsts) const; 250 251 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I) const; 253 254 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I) const; 256 257 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I) const; 259 260 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 261 MachineInstr &I, unsigned Opcode) const; 262 263 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 264 MachineInstr &I) const; 265 266 bool selectUnmergeValues(MachineInstr &I) const; 267 268 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 269 MachineInstr &I) const; 270 271 void selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 272 MachineInstr &I) const; 273 274 void selectImageWriteIntrinsic(MachineInstr &I) const; 275 276 // Utilities 277 std::pair<Register, bool> 278 buildI32Constant(uint32_t Val, MachineInstr &I, 279 const SPIRVType *ResType = nullptr) const; 280 281 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 282 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 283 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 284 MachineInstr &I) const; 285 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 286 287 bool wrapIntoSpecConstantOp(MachineInstr &I, 288 SmallVector<Register> &CompositeArgs) const; 289 290 Register getUcharPtrTypeReg(MachineInstr &I, 291 SPIRV::StorageClass::StorageClass SC) const; 292 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 293 Register Src, Register DestType, 294 uint32_t Opcode) const; 295 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 296 SPIRVType *SrcPtrTy) const; 297 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 298 uint32_t Binding, uint32_t ArraySize, 299 Register IndexReg, bool IsNonUniform, 300 MachineIRBuilder MIRBuilder) const; 301 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 302 void extractSubvector(Register &ResVReg, const SPIRVType *ResType, 303 Register &ReadReg, MachineInstr &InsertionPoint) const; 304 }; 305 306 } // end anonymous namespace 307 308 #define GET_GLOBALISEL_IMPL 309 #include "SPIRVGenGlobalISel.inc" 310 #undef GET_GLOBALISEL_IMPL 311 312 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 313 const SPIRVSubtarget &ST, 314 const RegisterBankInfo &RBI) 315 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 316 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 317 #define GET_GLOBALISEL_PREDICATES_INIT 318 #include "SPIRVGenGlobalISel.inc" 319 #undef GET_GLOBALISEL_PREDICATES_INIT 320 #define GET_GLOBALISEL_TEMPORARIES_INIT 321 #include "SPIRVGenGlobalISel.inc" 322 #undef GET_GLOBALISEL_TEMPORARIES_INIT 323 { 324 } 325 326 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 327 CodeGenCoverage *CoverageInfo, 328 ProfileSummaryInfo *PSI, 329 BlockFrequencyInfo *BFI) { 330 MRI = &MF.getRegInfo(); 331 GR.setCurrentFunc(MF); 332 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 333 } 334 335 // Ensure that register classes correspond to pattern matching rules. 336 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 337 if (HasVRegsReset == &MF) 338 return; 339 HasVRegsReset = &MF; 340 341 MachineRegisterInfo &MRI = MF.getRegInfo(); 342 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 343 Register Reg = Register::index2VirtReg(I); 344 LLT RegType = MRI.getType(Reg); 345 if (RegType.isScalar()) 346 MRI.setType(Reg, LLT::scalar(64)); 347 else if (RegType.isPointer()) 348 MRI.setType(Reg, LLT::pointer(0, 64)); 349 else if (RegType.isVector()) 350 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 351 } 352 for (const auto &MBB : MF) { 353 for (const auto &MI : MBB) { 354 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 355 continue; 356 Register DstReg = MI.getOperand(0).getReg(); 357 LLT DstType = MRI.getType(DstReg); 358 Register SrcReg = MI.getOperand(1).getReg(); 359 LLT SrcType = MRI.getType(SrcReg); 360 if (DstType != SrcType) 361 MRI.setType(DstReg, MRI.getType(SrcReg)); 362 363 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 364 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 365 if (DstRC != SrcRC && SrcRC) 366 MRI.setRegClass(DstReg, SrcRC); 367 } 368 } 369 } 370 371 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 372 373 // Defined in SPIRVLegalizerInfo.cpp. 374 extern bool isTypeFoldingSupported(unsigned Opcode); 375 376 bool SPIRVInstructionSelector::select(MachineInstr &I) { 377 resetVRegsType(*I.getParent()->getParent()); 378 379 assert(I.getParent() && "Instruction should be in a basic block!"); 380 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 381 382 Register Opcode = I.getOpcode(); 383 // If it's not a GMIR instruction, we've selected it already. 384 if (!isPreISelGenericOpcode(Opcode)) { 385 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 386 Register DstReg = I.getOperand(0).getReg(); 387 Register SrcReg = I.getOperand(1).getReg(); 388 auto *Def = MRI->getVRegDef(SrcReg); 389 if (isTypeFoldingSupported(Def->getOpcode())) { 390 bool Res = selectImpl(I, *CoverageInfo); 391 LLVM_DEBUG({ 392 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 393 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 394 I.print(dbgs()); 395 } 396 }); 397 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 398 if (Res) 399 return Res; 400 } 401 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 402 MRI->replaceRegWith(SrcReg, DstReg); 403 I.removeFromParent(); 404 return true; 405 } else if (I.getNumDefs() == 1) { 406 // Make all vregs 64 bits (for SPIR-V IDs). 407 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 408 } 409 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 410 } 411 412 if (I.getNumOperands() != I.getNumExplicitOperands()) { 413 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 414 return false; 415 } 416 417 // Common code for getting return reg+type, and removing selected instr 418 // from parent occurs here. Instr-specific selection happens in spvSelect(). 419 bool HasDefs = I.getNumDefs() > 0; 420 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 421 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 422 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 423 if (spvSelect(ResVReg, ResType, I)) { 424 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 425 for (unsigned i = 0; i < I.getNumDefs(); ++i) 426 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 427 I.removeFromParent(); 428 return true; 429 } 430 return false; 431 } 432 433 static bool mayApplyGenericSelection(unsigned Opcode) { 434 switch (Opcode) { 435 case TargetOpcode::G_CONSTANT: 436 return false; 437 case TargetOpcode::G_SADDO: 438 case TargetOpcode::G_SSUBO: 439 return true; 440 } 441 return isTypeFoldingSupported(Opcode); 442 } 443 444 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 445 const SPIRVType *ResType, 446 MachineInstr &I) const { 447 const unsigned Opcode = I.getOpcode(); 448 if (mayApplyGenericSelection(Opcode)) 449 return selectImpl(I, *CoverageInfo); 450 switch (Opcode) { 451 case TargetOpcode::G_CONSTANT: 452 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 453 I); 454 case TargetOpcode::G_GLOBAL_VALUE: 455 return selectGlobalValue(ResVReg, I); 456 case TargetOpcode::G_IMPLICIT_DEF: 457 return selectOpUndef(ResVReg, ResType, I); 458 case TargetOpcode::G_FREEZE: 459 return selectFreeze(ResVReg, ResType, I); 460 461 case TargetOpcode::G_INTRINSIC: 462 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 463 case TargetOpcode::G_INTRINSIC_CONVERGENT: 464 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 465 return selectIntrinsic(ResVReg, ResType, I); 466 case TargetOpcode::G_BITREVERSE: 467 return selectBitreverse(ResVReg, ResType, I); 468 469 case TargetOpcode::G_BUILD_VECTOR: 470 return selectBuildVector(ResVReg, ResType, I); 471 case TargetOpcode::G_SPLAT_VECTOR: 472 return selectSplatVector(ResVReg, ResType, I); 473 474 case TargetOpcode::G_SHUFFLE_VECTOR: { 475 MachineBasicBlock &BB = *I.getParent(); 476 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 477 .addDef(ResVReg) 478 .addUse(GR.getSPIRVTypeID(ResType)) 479 .addUse(I.getOperand(1).getReg()) 480 .addUse(I.getOperand(2).getReg()); 481 for (auto V : I.getOperand(3).getShuffleMask()) 482 MIB.addImm(V); 483 return MIB.constrainAllUses(TII, TRI, RBI); 484 } 485 case TargetOpcode::G_MEMMOVE: 486 case TargetOpcode::G_MEMCPY: 487 case TargetOpcode::G_MEMSET: 488 return selectMemOperation(ResVReg, I); 489 490 case TargetOpcode::G_ICMP: 491 return selectICmp(ResVReg, ResType, I); 492 case TargetOpcode::G_FCMP: 493 return selectFCmp(ResVReg, ResType, I); 494 495 case TargetOpcode::G_FRAME_INDEX: 496 return selectFrameIndex(ResVReg, ResType, I); 497 498 case TargetOpcode::G_LOAD: 499 return selectLoad(ResVReg, ResType, I); 500 case TargetOpcode::G_STORE: 501 return selectStore(I); 502 503 case TargetOpcode::G_BR: 504 return selectBranch(I); 505 case TargetOpcode::G_BRCOND: 506 return selectBranchCond(I); 507 508 case TargetOpcode::G_PHI: 509 return selectPhi(ResVReg, ResType, I); 510 511 case TargetOpcode::G_FPTOSI: 512 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 513 case TargetOpcode::G_FPTOUI: 514 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 515 516 case TargetOpcode::G_SITOFP: 517 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 518 case TargetOpcode::G_UITOFP: 519 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 520 521 case TargetOpcode::G_CTPOP: 522 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 523 case TargetOpcode::G_SMIN: 524 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 525 case TargetOpcode::G_UMIN: 526 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 527 528 case TargetOpcode::G_SMAX: 529 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 530 case TargetOpcode::G_UMAX: 531 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 532 533 case TargetOpcode::G_FMA: 534 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 535 536 case TargetOpcode::G_FPOW: 537 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 538 case TargetOpcode::G_FPOWI: 539 return selectExtInst(ResVReg, ResType, I, CL::pown); 540 541 case TargetOpcode::G_FEXP: 542 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 543 case TargetOpcode::G_FEXP2: 544 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 545 546 case TargetOpcode::G_FLOG: 547 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 548 case TargetOpcode::G_FLOG2: 549 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 550 case TargetOpcode::G_FLOG10: 551 return selectLog10(ResVReg, ResType, I); 552 553 case TargetOpcode::G_FABS: 554 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 555 case TargetOpcode::G_ABS: 556 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 557 558 case TargetOpcode::G_FMINNUM: 559 case TargetOpcode::G_FMINIMUM: 560 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 561 case TargetOpcode::G_FMAXNUM: 562 case TargetOpcode::G_FMAXIMUM: 563 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 564 565 case TargetOpcode::G_FCOPYSIGN: 566 return selectExtInst(ResVReg, ResType, I, CL::copysign); 567 568 case TargetOpcode::G_FCEIL: 569 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 570 case TargetOpcode::G_FFLOOR: 571 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 572 573 case TargetOpcode::G_FCOS: 574 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 575 case TargetOpcode::G_FSIN: 576 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 577 case TargetOpcode::G_FTAN: 578 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 579 case TargetOpcode::G_FACOS: 580 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 581 case TargetOpcode::G_FASIN: 582 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 583 case TargetOpcode::G_FATAN: 584 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 585 case TargetOpcode::G_FATAN2: 586 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 587 case TargetOpcode::G_FCOSH: 588 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 589 case TargetOpcode::G_FSINH: 590 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 591 case TargetOpcode::G_FTANH: 592 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 593 594 case TargetOpcode::G_FSQRT: 595 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 596 597 case TargetOpcode::G_CTTZ: 598 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 599 return selectExtInst(ResVReg, ResType, I, CL::ctz); 600 case TargetOpcode::G_CTLZ: 601 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 602 return selectExtInst(ResVReg, ResType, I, CL::clz); 603 604 case TargetOpcode::G_INTRINSIC_ROUND: 605 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 606 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 607 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 608 case TargetOpcode::G_INTRINSIC_TRUNC: 609 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 610 case TargetOpcode::G_FRINT: 611 case TargetOpcode::G_FNEARBYINT: 612 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 613 614 case TargetOpcode::G_SMULH: 615 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 616 case TargetOpcode::G_UMULH: 617 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 618 619 case TargetOpcode::G_SADDSAT: 620 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 621 case TargetOpcode::G_UADDSAT: 622 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 623 case TargetOpcode::G_SSUBSAT: 624 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 625 case TargetOpcode::G_USUBSAT: 626 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 627 628 case TargetOpcode::G_UADDO: 629 return selectOverflowArith(ResVReg, ResType, I, 630 ResType->getOpcode() == SPIRV::OpTypeVector 631 ? SPIRV::OpIAddCarryV 632 : SPIRV::OpIAddCarryS); 633 case TargetOpcode::G_USUBO: 634 return selectOverflowArith(ResVReg, ResType, I, 635 ResType->getOpcode() == SPIRV::OpTypeVector 636 ? SPIRV::OpISubBorrowV 637 : SPIRV::OpISubBorrowS); 638 case TargetOpcode::G_UMULO: 639 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 640 case TargetOpcode::G_SMULO: 641 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 642 643 case TargetOpcode::G_SEXT: 644 return selectExt(ResVReg, ResType, I, true); 645 case TargetOpcode::G_ANYEXT: 646 case TargetOpcode::G_ZEXT: 647 return selectExt(ResVReg, ResType, I, false); 648 case TargetOpcode::G_TRUNC: 649 return selectTrunc(ResVReg, ResType, I); 650 case TargetOpcode::G_FPTRUNC: 651 case TargetOpcode::G_FPEXT: 652 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 653 654 case TargetOpcode::G_PTRTOINT: 655 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 656 case TargetOpcode::G_INTTOPTR: 657 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 658 case TargetOpcode::G_BITCAST: 659 return selectBitcast(ResVReg, ResType, I); 660 case TargetOpcode::G_ADDRSPACE_CAST: 661 return selectAddrSpaceCast(ResVReg, ResType, I); 662 case TargetOpcode::G_PTR_ADD: { 663 // Currently, we get G_PTR_ADD only applied to global variables. 664 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 665 Register GV = I.getOperand(1).getReg(); 666 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 667 (void)II; 668 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 669 (*II).getOpcode() == TargetOpcode::COPY || 670 (*II).getOpcode() == SPIRV::OpVariable) && 671 isImm(I.getOperand(2), MRI)); 672 // It may be the initialization of a global variable. 673 bool IsGVInit = false; 674 for (MachineRegisterInfo::use_instr_iterator 675 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 676 UseEnd = MRI->use_instr_end(); 677 UseIt != UseEnd; UseIt = std::next(UseIt)) { 678 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 679 (*UseIt).getOpcode() == SPIRV::OpVariable) { 680 IsGVInit = true; 681 break; 682 } 683 } 684 MachineBasicBlock &BB = *I.getParent(); 685 if (!IsGVInit) { 686 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 687 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 688 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 689 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 690 // Build a new virtual register that is associated with the required 691 // data type. 692 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 693 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 694 // Having a correctly typed base we are ready to build the actually 695 // required GEP. It may not be a constant though, because all Operands 696 // of OpSpecConstantOp is to originate from other const instructions, 697 // and only the AccessChain named opcodes accept a global OpVariable 698 // instruction. We can't use an AccessChain opcode because of the type 699 // mismatch between result and base types. 700 if (!GR.isBitcastCompatible(ResType, GVType)) 701 report_fatal_error( 702 "incompatible result and operand types in a bitcast"); 703 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 704 MachineInstrBuilder MIB = 705 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 706 .addDef(NewVReg) 707 .addUse(ResTypeReg) 708 .addUse(GV); 709 return MIB.constrainAllUses(TII, TRI, RBI) && 710 BuildMI(BB, I, I.getDebugLoc(), 711 TII.get(STI.isVulkanEnv() 712 ? SPIRV::OpInBoundsAccessChain 713 : SPIRV::OpInBoundsPtrAccessChain)) 714 .addDef(ResVReg) 715 .addUse(ResTypeReg) 716 .addUse(NewVReg) 717 .addUse(I.getOperand(2).getReg()) 718 .constrainAllUses(TII, TRI, RBI); 719 } else { 720 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 721 .addDef(ResVReg) 722 .addUse(GR.getSPIRVTypeID(ResType)) 723 .addImm( 724 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 725 .addUse(GV) 726 .addUse(I.getOperand(2).getReg()) 727 .constrainAllUses(TII, TRI, RBI); 728 } 729 } 730 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 731 // initialize a global variable with a constant expression (e.g., the test 732 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 733 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 734 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 735 .addDef(ResVReg) 736 .addUse(GR.getSPIRVTypeID(ResType)) 737 .addImm(static_cast<uint32_t>( 738 SPIRV::Opcode::InBoundsPtrAccessChain)) 739 .addUse(GV) 740 .addUse(Idx) 741 .addUse(I.getOperand(2).getReg()); 742 return MIB.constrainAllUses(TII, TRI, RBI); 743 } 744 745 case TargetOpcode::G_ATOMICRMW_OR: 746 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 747 case TargetOpcode::G_ATOMICRMW_ADD: 748 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 749 case TargetOpcode::G_ATOMICRMW_AND: 750 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 751 case TargetOpcode::G_ATOMICRMW_MAX: 752 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 753 case TargetOpcode::G_ATOMICRMW_MIN: 754 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 755 case TargetOpcode::G_ATOMICRMW_SUB: 756 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 757 case TargetOpcode::G_ATOMICRMW_XOR: 758 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 759 case TargetOpcode::G_ATOMICRMW_UMAX: 760 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 761 case TargetOpcode::G_ATOMICRMW_UMIN: 762 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 763 case TargetOpcode::G_ATOMICRMW_XCHG: 764 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 765 case TargetOpcode::G_ATOMIC_CMPXCHG: 766 return selectAtomicCmpXchg(ResVReg, ResType, I); 767 768 case TargetOpcode::G_ATOMICRMW_FADD: 769 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 770 case TargetOpcode::G_ATOMICRMW_FSUB: 771 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 772 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 773 SPIRV::OpFNegate); 774 case TargetOpcode::G_ATOMICRMW_FMIN: 775 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 776 case TargetOpcode::G_ATOMICRMW_FMAX: 777 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 778 779 case TargetOpcode::G_FENCE: 780 return selectFence(I); 781 782 case TargetOpcode::G_STACKSAVE: 783 return selectStackSave(ResVReg, ResType, I); 784 case TargetOpcode::G_STACKRESTORE: 785 return selectStackRestore(I); 786 787 case TargetOpcode::G_UNMERGE_VALUES: 788 return selectUnmergeValues(I); 789 790 // Discard gen opcodes for intrinsics which we do not expect to actually 791 // represent code after lowering or intrinsics which are not implemented but 792 // should not crash when found in a customer's LLVM IR input. 793 case TargetOpcode::G_TRAP: 794 case TargetOpcode::G_DEBUGTRAP: 795 case TargetOpcode::G_UBSANTRAP: 796 case TargetOpcode::DBG_LABEL: 797 return true; 798 799 default: 800 return false; 801 } 802 } 803 804 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 805 const SPIRVType *ResType, 806 MachineInstr &I, 807 GL::GLSLExtInst GLInst) const { 808 return selectExtInst(ResVReg, ResType, I, 809 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 810 } 811 812 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 813 const SPIRVType *ResType, 814 MachineInstr &I, 815 CL::OpenCLExtInst CLInst) const { 816 return selectExtInst(ResVReg, ResType, I, 817 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 818 } 819 820 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 821 const SPIRVType *ResType, 822 MachineInstr &I, 823 CL::OpenCLExtInst CLInst, 824 GL::GLSLExtInst GLInst) const { 825 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 826 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 827 return selectExtInst(ResVReg, ResType, I, ExtInsts); 828 } 829 830 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 831 const SPIRVType *ResType, 832 MachineInstr &I, 833 const ExtInstList &Insts) const { 834 835 for (const auto &Ex : Insts) { 836 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 837 uint32_t Opcode = Ex.second; 838 if (STI.canUseExtInstSet(Set)) { 839 MachineBasicBlock &BB = *I.getParent(); 840 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 841 .addDef(ResVReg) 842 .addUse(GR.getSPIRVTypeID(ResType)) 843 .addImm(static_cast<uint32_t>(Set)) 844 .addImm(Opcode); 845 const unsigned NumOps = I.getNumOperands(); 846 unsigned Index = 1; 847 if (Index < NumOps && 848 I.getOperand(Index).getType() == 849 MachineOperand::MachineOperandType::MO_IntrinsicID) 850 Index = 2; 851 for (; Index < NumOps; ++Index) 852 MIB.add(I.getOperand(Index)); 853 return MIB.constrainAllUses(TII, TRI, RBI); 854 } 855 } 856 return false; 857 } 858 859 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 860 const SPIRVType *ResType, 861 MachineInstr &I, 862 std::vector<Register> Srcs, 863 unsigned Opcode) const { 864 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 865 .addDef(ResVReg) 866 .addUse(GR.getSPIRVTypeID(ResType)); 867 for (Register SReg : Srcs) { 868 MIB.addUse(SReg); 869 } 870 return MIB.constrainAllUses(TII, TRI, RBI); 871 } 872 873 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 874 const SPIRVType *ResType, 875 MachineInstr &I, 876 unsigned Opcode) const { 877 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 878 Register SrcReg = I.getOperand(1).getReg(); 879 bool IsGV = false; 880 for (MachineRegisterInfo::def_instr_iterator DefIt = 881 MRI->def_instr_begin(SrcReg); 882 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 883 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 884 IsGV = true; 885 break; 886 } 887 } 888 if (IsGV) { 889 uint32_t SpecOpcode = 0; 890 switch (Opcode) { 891 case SPIRV::OpConvertPtrToU: 892 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 893 break; 894 case SPIRV::OpConvertUToPtr: 895 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 896 break; 897 } 898 if (SpecOpcode) 899 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 900 TII.get(SPIRV::OpSpecConstantOp)) 901 .addDef(ResVReg) 902 .addUse(GR.getSPIRVTypeID(ResType)) 903 .addImm(SpecOpcode) 904 .addUse(SrcReg) 905 .constrainAllUses(TII, TRI, RBI); 906 } 907 } 908 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 909 Opcode); 910 } 911 912 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 913 const SPIRVType *ResType, 914 MachineInstr &I) const { 915 Register OpReg = I.getOperand(1).getReg(); 916 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 917 if (!GR.isBitcastCompatible(ResType, OpType)) 918 report_fatal_error("incompatible result and operand types in a bitcast"); 919 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 920 } 921 922 static void addMemoryOperands(MachineMemOperand *MemOp, 923 MachineInstrBuilder &MIB) { 924 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 925 if (MemOp->isVolatile()) 926 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 927 if (MemOp->isNonTemporal()) 928 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 929 if (MemOp->getAlign().value()) 930 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 931 932 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 933 MIB.addImm(SpvMemOp); 934 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 935 MIB.addImm(MemOp->getAlign().value()); 936 } 937 } 938 939 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 940 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 941 if (Flags & MachineMemOperand::Flags::MOVolatile) 942 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 943 if (Flags & MachineMemOperand::Flags::MONonTemporal) 944 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 945 946 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 947 MIB.addImm(SpvMemOp); 948 } 949 950 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 951 const SPIRVType *ResType, 952 MachineInstr &I) const { 953 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 954 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 955 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 956 .addDef(ResVReg) 957 .addUse(GR.getSPIRVTypeID(ResType)) 958 .addUse(Ptr); 959 if (!I.getNumMemOperands()) { 960 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 961 I.getOpcode() == 962 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 963 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 964 } else { 965 addMemoryOperands(*I.memoperands_begin(), MIB); 966 } 967 return MIB.constrainAllUses(TII, TRI, RBI); 968 } 969 970 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 971 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 972 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 973 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 974 MachineBasicBlock &BB = *I.getParent(); 975 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 976 .addUse(Ptr) 977 .addUse(StoreVal); 978 if (!I.getNumMemOperands()) { 979 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 980 I.getOpcode() == 981 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 982 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 983 } else { 984 addMemoryOperands(*I.memoperands_begin(), MIB); 985 } 986 return MIB.constrainAllUses(TII, TRI, RBI); 987 } 988 989 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 990 const SPIRVType *ResType, 991 MachineInstr &I) const { 992 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 993 report_fatal_error( 994 "llvm.stacksave intrinsic: this instruction requires the following " 995 "SPIR-V extension: SPV_INTEL_variable_length_array", 996 false); 997 MachineBasicBlock &BB = *I.getParent(); 998 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 999 .addDef(ResVReg) 1000 .addUse(GR.getSPIRVTypeID(ResType)) 1001 .constrainAllUses(TII, TRI, RBI); 1002 } 1003 1004 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1005 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1006 report_fatal_error( 1007 "llvm.stackrestore intrinsic: this instruction requires the following " 1008 "SPIR-V extension: SPV_INTEL_variable_length_array", 1009 false); 1010 if (!I.getOperand(0).isReg()) 1011 return false; 1012 MachineBasicBlock &BB = *I.getParent(); 1013 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1014 .addUse(I.getOperand(0).getReg()) 1015 .constrainAllUses(TII, TRI, RBI); 1016 } 1017 1018 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1019 MachineInstr &I) const { 1020 MachineBasicBlock &BB = *I.getParent(); 1021 Register SrcReg = I.getOperand(1).getReg(); 1022 bool Result = true; 1023 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1024 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1025 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1026 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1027 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1028 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1029 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1030 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1031 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1032 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1033 Function &CurFunction = GR.CurMF->getFunction(); 1034 Type *LLVMArrTy = 1035 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1036 // Module takes ownership of the global var. 1037 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1038 true, GlobalValue::InternalLinkage, 1039 Constant::getNullValue(LLVMArrTy)); 1040 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1041 GR.add(GV, GR.CurMF, VarReg); 1042 1043 Result &= 1044 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1045 .addDef(VarReg) 1046 .addUse(GR.getSPIRVTypeID(VarTy)) 1047 .addImm(SPIRV::StorageClass::UniformConstant) 1048 .addUse(Const) 1049 .constrainAllUses(TII, TRI, RBI); 1050 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1051 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1052 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1053 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1054 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1055 } 1056 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1057 .addUse(I.getOperand(0).getReg()) 1058 .addUse(SrcReg) 1059 .addUse(I.getOperand(2).getReg()); 1060 if (I.getNumMemOperands()) 1061 addMemoryOperands(*I.memoperands_begin(), MIB); 1062 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1063 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1064 Result &= 1065 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1066 .addUse(MIB->getOperand(0).getReg()) 1067 .constrainAllUses(TII, TRI, RBI); 1068 return Result; 1069 } 1070 1071 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1072 const SPIRVType *ResType, 1073 MachineInstr &I, 1074 unsigned NewOpcode, 1075 unsigned NegateOpcode) const { 1076 bool Result = true; 1077 assert(I.hasOneMemOperand()); 1078 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1079 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1080 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1081 auto ScopeConstant = buildI32Constant(Scope, I); 1082 Register ScopeReg = ScopeConstant.first; 1083 Result &= ScopeConstant.second; 1084 1085 Register Ptr = I.getOperand(1).getReg(); 1086 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1087 // auto ScSem = 1088 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1089 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1090 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1091 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1092 Register MemSemReg = MemSemConstant.first; 1093 Result &= MemSemConstant.second; 1094 1095 Register ValueReg = I.getOperand(2).getReg(); 1096 if (NegateOpcode != 0) { 1097 // Translation with negative value operand is requested 1098 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1099 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1100 ValueReg = TmpReg; 1101 } 1102 1103 return Result && 1104 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1105 .addDef(ResVReg) 1106 .addUse(GR.getSPIRVTypeID(ResType)) 1107 .addUse(Ptr) 1108 .addUse(ScopeReg) 1109 .addUse(MemSemReg) 1110 .addUse(ValueReg) 1111 .constrainAllUses(TII, TRI, RBI); 1112 } 1113 1114 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1115 unsigned ArgI = I.getNumOperands() - 1; 1116 Register SrcReg = 1117 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1118 SPIRVType *DefType = 1119 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1120 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1121 report_fatal_error( 1122 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1123 1124 SPIRVType *ScalarType = 1125 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1126 MachineBasicBlock &BB = *I.getParent(); 1127 bool Res = false; 1128 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1129 Register ResVReg = I.getOperand(i).getReg(); 1130 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1131 if (!ResType) { 1132 // There was no "assign type" actions, let's fix this now 1133 ResType = ScalarType; 1134 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1135 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1136 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1137 } 1138 auto MIB = 1139 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1140 .addDef(ResVReg) 1141 .addUse(GR.getSPIRVTypeID(ResType)) 1142 .addUse(SrcReg) 1143 .addImm(static_cast<int64_t>(i)); 1144 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1145 } 1146 return Res; 1147 } 1148 1149 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1150 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1151 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1152 auto MemSemConstant = buildI32Constant(MemSem, I); 1153 Register MemSemReg = MemSemConstant.first; 1154 bool Result = MemSemConstant.second; 1155 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1156 uint32_t Scope = static_cast<uint32_t>( 1157 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1158 auto ScopeConstant = buildI32Constant(Scope, I); 1159 Register ScopeReg = ScopeConstant.first; 1160 Result &= ScopeConstant.second; 1161 MachineBasicBlock &BB = *I.getParent(); 1162 return Result && 1163 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1164 .addUse(ScopeReg) 1165 .addUse(MemSemReg) 1166 .constrainAllUses(TII, TRI, RBI); 1167 } 1168 1169 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1170 const SPIRVType *ResType, 1171 MachineInstr &I, 1172 unsigned Opcode) const { 1173 Type *ResTy = nullptr; 1174 StringRef ResName; 1175 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1176 report_fatal_error( 1177 "Not enough info to select the arithmetic with overflow instruction"); 1178 if (!ResTy || !ResTy->isStructTy()) 1179 report_fatal_error("Expect struct type result for the arithmetic " 1180 "with overflow instruction"); 1181 // "Result Type must be from OpTypeStruct. The struct must have two members, 1182 // and the two members must be the same type." 1183 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1184 ResTy = StructType::get(ResElemTy, ResElemTy); 1185 // Build SPIR-V types and constant(s) if needed. 1186 MachineIRBuilder MIRBuilder(I); 1187 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1188 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1189 assert(I.getNumDefs() > 1 && "Not enought operands"); 1190 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1191 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1192 if (N > 1) 1193 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1194 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1195 Register ZeroReg = buildZerosVal(ResType, I); 1196 // A new virtual register to store the result struct. 1197 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1198 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1199 // Build the result name if needed. 1200 if (ResName.size() > 0) 1201 buildOpName(StructVReg, ResName, MIRBuilder); 1202 // Build the arithmetic with overflow instruction. 1203 MachineBasicBlock &BB = *I.getParent(); 1204 auto MIB = 1205 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1206 .addDef(StructVReg) 1207 .addUse(GR.getSPIRVTypeID(StructType)); 1208 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1209 MIB.addUse(I.getOperand(i).getReg()); 1210 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1211 // Build instructions to extract fields of the instruction's result. 1212 // A new virtual register to store the higher part of the result struct. 1213 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1214 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1215 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1216 auto MIB = 1217 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1218 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1219 .addUse(GR.getSPIRVTypeID(ResType)) 1220 .addUse(StructVReg) 1221 .addImm(i); 1222 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1223 } 1224 // Build boolean value from the higher part. 1225 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1226 .addDef(I.getOperand(1).getReg()) 1227 .addUse(BoolTypeReg) 1228 .addUse(HigherVReg) 1229 .addUse(ZeroReg) 1230 .constrainAllUses(TII, TRI, RBI); 1231 } 1232 1233 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1234 const SPIRVType *ResType, 1235 MachineInstr &I) const { 1236 bool Result = true; 1237 Register ScopeReg; 1238 Register MemSemEqReg; 1239 Register MemSemNeqReg; 1240 Register Ptr = I.getOperand(2).getReg(); 1241 if (!isa<GIntrinsic>(I)) { 1242 assert(I.hasOneMemOperand()); 1243 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1244 unsigned Scope = static_cast<uint32_t>(getMemScope( 1245 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1246 auto ScopeConstant = buildI32Constant(Scope, I); 1247 ScopeReg = ScopeConstant.first; 1248 Result &= ScopeConstant.second; 1249 1250 unsigned ScSem = static_cast<uint32_t>( 1251 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1252 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1253 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1254 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1255 MemSemEqReg = MemSemEqConstant.first; 1256 Result &= MemSemEqConstant.second; 1257 AtomicOrdering FO = MemOp->getFailureOrdering(); 1258 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1259 if (MemSemEq == MemSemNeq) 1260 MemSemNeqReg = MemSemEqReg; 1261 else { 1262 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1263 MemSemNeqReg = MemSemNeqConstant.first; 1264 Result &= MemSemNeqConstant.second; 1265 } 1266 } else { 1267 ScopeReg = I.getOperand(5).getReg(); 1268 MemSemEqReg = I.getOperand(6).getReg(); 1269 MemSemNeqReg = I.getOperand(7).getReg(); 1270 } 1271 1272 Register Cmp = I.getOperand(3).getReg(); 1273 Register Val = I.getOperand(4).getReg(); 1274 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1275 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1276 const DebugLoc &DL = I.getDebugLoc(); 1277 Result &= 1278 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1279 .addDef(ACmpRes) 1280 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1281 .addUse(Ptr) 1282 .addUse(ScopeReg) 1283 .addUse(MemSemEqReg) 1284 .addUse(MemSemNeqReg) 1285 .addUse(Val) 1286 .addUse(Cmp) 1287 .constrainAllUses(TII, TRI, RBI); 1288 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1289 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1290 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1291 .addDef(CmpSuccReg) 1292 .addUse(GR.getSPIRVTypeID(BoolTy)) 1293 .addUse(ACmpRes) 1294 .addUse(Cmp) 1295 .constrainAllUses(TII, TRI, RBI); 1296 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1297 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1298 .addDef(TmpReg) 1299 .addUse(GR.getSPIRVTypeID(ResType)) 1300 .addUse(ACmpRes) 1301 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1302 .addImm(0) 1303 .constrainAllUses(TII, TRI, RBI); 1304 return Result && 1305 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1306 .addDef(ResVReg) 1307 .addUse(GR.getSPIRVTypeID(ResType)) 1308 .addUse(CmpSuccReg) 1309 .addUse(TmpReg) 1310 .addImm(1) 1311 .constrainAllUses(TII, TRI, RBI); 1312 } 1313 1314 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1315 switch (SC) { 1316 case SPIRV::StorageClass::Workgroup: 1317 case SPIRV::StorageClass::CrossWorkgroup: 1318 case SPIRV::StorageClass::Function: 1319 return true; 1320 default: 1321 return false; 1322 } 1323 } 1324 1325 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1326 switch (SC) { 1327 case SPIRV::StorageClass::DeviceOnlyINTEL: 1328 case SPIRV::StorageClass::HostOnlyINTEL: 1329 return true; 1330 default: 1331 return false; 1332 } 1333 } 1334 1335 // Returns true ResVReg is referred only from global vars and OpName's. 1336 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1337 bool IsGRef = false; 1338 bool IsAllowedRefs = 1339 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1340 [&IsGRef](auto const &It) { 1341 unsigned Opcode = It.getOpcode(); 1342 if (Opcode == SPIRV::OpConstantComposite || 1343 Opcode == SPIRV::OpVariable || 1344 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1345 return IsGRef = true; 1346 return Opcode == SPIRV::OpName; 1347 }); 1348 return IsAllowedRefs && IsGRef; 1349 } 1350 1351 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1352 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1353 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1354 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1355 } 1356 1357 MachineInstrBuilder 1358 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1359 Register Src, Register DestType, 1360 uint32_t Opcode) const { 1361 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1362 TII.get(SPIRV::OpSpecConstantOp)) 1363 .addDef(Dest) 1364 .addUse(DestType) 1365 .addImm(Opcode) 1366 .addUse(Src); 1367 } 1368 1369 MachineInstrBuilder 1370 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1371 SPIRVType *SrcPtrTy) const { 1372 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1373 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1374 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1375 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1376 SPIRV::StorageClass::Generic), 1377 GR.getPointerSize())); 1378 MachineFunction *MF = I.getParent()->getParent(); 1379 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1380 MachineInstrBuilder MIB = buildSpecConstantOp( 1381 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1382 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1383 GR.add(MIB.getInstr(), MF, Tmp); 1384 return MIB; 1385 } 1386 1387 // In SPIR-V address space casting can only happen to and from the Generic 1388 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1389 // pointers to and from Generic pointers. As such, we can convert e.g. from 1390 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1391 // other combinations can only be done by a bitcast, and are probably not safe. 1392 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1393 const SPIRVType *ResType, 1394 MachineInstr &I) const { 1395 MachineBasicBlock &BB = *I.getParent(); 1396 const DebugLoc &DL = I.getDebugLoc(); 1397 1398 Register SrcPtr = I.getOperand(1).getReg(); 1399 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1400 1401 // don't generate a cast for a null that may be represented by OpTypeInt 1402 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1403 ResType->getOpcode() != SPIRV::OpTypePointer) 1404 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1405 .addDef(ResVReg) 1406 .addUse(SrcPtr) 1407 .constrainAllUses(TII, TRI, RBI); 1408 1409 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1410 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1411 1412 if (isASCastInGVar(MRI, ResVReg)) { 1413 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1414 // are expressed by OpSpecConstantOp with an Opcode. 1415 // TODO: maybe insert a check whether the Kernel capability was declared and 1416 // so PtrCastToGeneric/GenericCastToPtr are available. 1417 unsigned SpecOpcode = 1418 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1419 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1420 : (SrcSC == SPIRV::StorageClass::Generic && 1421 isGenericCastablePtr(DstSC) 1422 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1423 : 0); 1424 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1425 // correct value of ResType and use general i8* instead. Maybe this should 1426 // be addressed in the emit-intrinsic step to infer a correct 1427 // OpConstantComposite type. 1428 if (SpecOpcode) { 1429 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1430 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1431 .constrainAllUses(TII, TRI, RBI); 1432 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1433 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1434 return MIB.constrainAllUses(TII, TRI, RBI) && 1435 buildSpecConstantOp( 1436 I, ResVReg, MIB->getOperand(0).getReg(), 1437 getUcharPtrTypeReg(I, DstSC), 1438 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1439 .constrainAllUses(TII, TRI, RBI); 1440 } 1441 } 1442 1443 // don't generate a cast between identical storage classes 1444 if (SrcSC == DstSC) 1445 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1446 .addDef(ResVReg) 1447 .addUse(SrcPtr) 1448 .constrainAllUses(TII, TRI, RBI); 1449 1450 // Casting from an eligible pointer to Generic. 1451 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1452 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1453 // Casting from Generic to an eligible pointer. 1454 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1455 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1456 // Casting between 2 eligible pointers using Generic as an intermediary. 1457 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1458 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1459 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1460 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1461 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1462 .addDef(Tmp) 1463 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1464 .addUse(SrcPtr) 1465 .constrainAllUses(TII, TRI, RBI); 1466 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1467 .addDef(ResVReg) 1468 .addUse(GR.getSPIRVTypeID(ResType)) 1469 .addUse(Tmp) 1470 .constrainAllUses(TII, TRI, RBI); 1471 } 1472 1473 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1474 // be applied 1475 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1476 return selectUnOp(ResVReg, ResType, I, 1477 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1478 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1479 return selectUnOp(ResVReg, ResType, I, 1480 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1481 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1482 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1483 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1484 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1485 1486 // Bitcast for pointers requires that the address spaces must match 1487 return false; 1488 } 1489 1490 static unsigned getFCmpOpcode(unsigned PredNum) { 1491 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1492 switch (Pred) { 1493 case CmpInst::FCMP_OEQ: 1494 return SPIRV::OpFOrdEqual; 1495 case CmpInst::FCMP_OGE: 1496 return SPIRV::OpFOrdGreaterThanEqual; 1497 case CmpInst::FCMP_OGT: 1498 return SPIRV::OpFOrdGreaterThan; 1499 case CmpInst::FCMP_OLE: 1500 return SPIRV::OpFOrdLessThanEqual; 1501 case CmpInst::FCMP_OLT: 1502 return SPIRV::OpFOrdLessThan; 1503 case CmpInst::FCMP_ONE: 1504 return SPIRV::OpFOrdNotEqual; 1505 case CmpInst::FCMP_ORD: 1506 return SPIRV::OpOrdered; 1507 case CmpInst::FCMP_UEQ: 1508 return SPIRV::OpFUnordEqual; 1509 case CmpInst::FCMP_UGE: 1510 return SPIRV::OpFUnordGreaterThanEqual; 1511 case CmpInst::FCMP_UGT: 1512 return SPIRV::OpFUnordGreaterThan; 1513 case CmpInst::FCMP_ULE: 1514 return SPIRV::OpFUnordLessThanEqual; 1515 case CmpInst::FCMP_ULT: 1516 return SPIRV::OpFUnordLessThan; 1517 case CmpInst::FCMP_UNE: 1518 return SPIRV::OpFUnordNotEqual; 1519 case CmpInst::FCMP_UNO: 1520 return SPIRV::OpUnordered; 1521 default: 1522 llvm_unreachable("Unknown predicate type for FCmp"); 1523 } 1524 } 1525 1526 static unsigned getICmpOpcode(unsigned PredNum) { 1527 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1528 switch (Pred) { 1529 case CmpInst::ICMP_EQ: 1530 return SPIRV::OpIEqual; 1531 case CmpInst::ICMP_NE: 1532 return SPIRV::OpINotEqual; 1533 case CmpInst::ICMP_SGE: 1534 return SPIRV::OpSGreaterThanEqual; 1535 case CmpInst::ICMP_SGT: 1536 return SPIRV::OpSGreaterThan; 1537 case CmpInst::ICMP_SLE: 1538 return SPIRV::OpSLessThanEqual; 1539 case CmpInst::ICMP_SLT: 1540 return SPIRV::OpSLessThan; 1541 case CmpInst::ICMP_UGE: 1542 return SPIRV::OpUGreaterThanEqual; 1543 case CmpInst::ICMP_UGT: 1544 return SPIRV::OpUGreaterThan; 1545 case CmpInst::ICMP_ULE: 1546 return SPIRV::OpULessThanEqual; 1547 case CmpInst::ICMP_ULT: 1548 return SPIRV::OpULessThan; 1549 default: 1550 llvm_unreachable("Unknown predicate type for ICmp"); 1551 } 1552 } 1553 1554 static unsigned getPtrCmpOpcode(unsigned Pred) { 1555 switch (static_cast<CmpInst::Predicate>(Pred)) { 1556 case CmpInst::ICMP_EQ: 1557 return SPIRV::OpPtrEqual; 1558 case CmpInst::ICMP_NE: 1559 return SPIRV::OpPtrNotEqual; 1560 default: 1561 llvm_unreachable("Unknown predicate type for pointer comparison"); 1562 } 1563 } 1564 1565 // Return the logical operation, or abort if none exists. 1566 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1567 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1568 switch (Pred) { 1569 case CmpInst::ICMP_EQ: 1570 return SPIRV::OpLogicalEqual; 1571 case CmpInst::ICMP_NE: 1572 return SPIRV::OpLogicalNotEqual; 1573 default: 1574 llvm_unreachable("Unknown predicate type for Bool comparison"); 1575 } 1576 } 1577 1578 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1579 if (!LLVMFloatTy) 1580 return APFloat::getZero(APFloat::IEEEsingle()); 1581 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1582 case Type::HalfTyID: 1583 return APFloat::getZero(APFloat::IEEEhalf()); 1584 default: 1585 case Type::FloatTyID: 1586 return APFloat::getZero(APFloat::IEEEsingle()); 1587 case Type::DoubleTyID: 1588 return APFloat::getZero(APFloat::IEEEdouble()); 1589 } 1590 } 1591 1592 static APFloat getOneFP(const Type *LLVMFloatTy) { 1593 if (!LLVMFloatTy) 1594 return APFloat::getOne(APFloat::IEEEsingle()); 1595 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1596 case Type::HalfTyID: 1597 return APFloat::getOne(APFloat::IEEEhalf()); 1598 default: 1599 case Type::FloatTyID: 1600 return APFloat::getOne(APFloat::IEEEsingle()); 1601 case Type::DoubleTyID: 1602 return APFloat::getOne(APFloat::IEEEdouble()); 1603 } 1604 } 1605 1606 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1607 const SPIRVType *ResType, 1608 MachineInstr &I, 1609 unsigned OpAnyOrAll) const { 1610 assert(I.getNumOperands() == 3); 1611 assert(I.getOperand(2).isReg()); 1612 MachineBasicBlock &BB = *I.getParent(); 1613 Register InputRegister = I.getOperand(2).getReg(); 1614 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1615 1616 if (!InputType) 1617 report_fatal_error("Input Type could not be determined."); 1618 1619 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1620 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1621 if (IsBoolTy && !IsVectorTy) { 1622 assert(ResVReg == I.getOperand(0).getReg()); 1623 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1624 TII.get(TargetOpcode::COPY)) 1625 .addDef(ResVReg) 1626 .addUse(InputRegister) 1627 .constrainAllUses(TII, TRI, RBI); 1628 } 1629 1630 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1631 unsigned SpirvNotEqualId = 1632 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1633 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1634 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1635 Register NotEqualReg = ResVReg; 1636 1637 if (IsVectorTy) { 1638 NotEqualReg = IsBoolTy ? InputRegister 1639 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1640 const unsigned NumElts = InputType->getOperand(2).getImm(); 1641 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1642 } 1643 1644 bool Result = true; 1645 if (!IsBoolTy) { 1646 Register ConstZeroReg = 1647 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1648 1649 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1650 .addDef(NotEqualReg) 1651 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1652 .addUse(InputRegister) 1653 .addUse(ConstZeroReg) 1654 .constrainAllUses(TII, TRI, RBI); 1655 } 1656 1657 if (!IsVectorTy) 1658 return Result; 1659 1660 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1661 .addDef(ResVReg) 1662 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1663 .addUse(NotEqualReg) 1664 .constrainAllUses(TII, TRI, RBI); 1665 } 1666 1667 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1668 const SPIRVType *ResType, 1669 MachineInstr &I) const { 1670 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1671 } 1672 1673 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1674 const SPIRVType *ResType, 1675 MachineInstr &I) const { 1676 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1677 } 1678 1679 // Select the OpDot instruction for the given float dot 1680 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1681 const SPIRVType *ResType, 1682 MachineInstr &I) const { 1683 assert(I.getNumOperands() == 4); 1684 assert(I.getOperand(2).isReg()); 1685 assert(I.getOperand(3).isReg()); 1686 1687 [[maybe_unused]] SPIRVType *VecType = 1688 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1689 1690 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1691 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1692 "dot product requires a vector of at least 2 components"); 1693 1694 [[maybe_unused]] SPIRVType *EltType = 1695 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1696 1697 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1698 1699 MachineBasicBlock &BB = *I.getParent(); 1700 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1701 .addDef(ResVReg) 1702 .addUse(GR.getSPIRVTypeID(ResType)) 1703 .addUse(I.getOperand(2).getReg()) 1704 .addUse(I.getOperand(3).getReg()) 1705 .constrainAllUses(TII, TRI, RBI); 1706 } 1707 1708 // Since pre-1.6 SPIRV has no integer dot implementation, 1709 // expand by piecewise multiplying and adding the results 1710 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1711 const SPIRVType *ResType, 1712 MachineInstr &I) const { 1713 assert(I.getNumOperands() == 4); 1714 assert(I.getOperand(2).isReg()); 1715 assert(I.getOperand(3).isReg()); 1716 MachineBasicBlock &BB = *I.getParent(); 1717 1718 // Multiply the vectors, then sum the results 1719 Register Vec0 = I.getOperand(2).getReg(); 1720 Register Vec1 = I.getOperand(3).getReg(); 1721 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1722 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1723 1724 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1725 .addDef(TmpVec) 1726 .addUse(GR.getSPIRVTypeID(VecType)) 1727 .addUse(Vec0) 1728 .addUse(Vec1) 1729 .constrainAllUses(TII, TRI, RBI); 1730 1731 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1732 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1733 "dot product requires a vector of at least 2 components"); 1734 1735 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1736 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1737 .addDef(Res) 1738 .addUse(GR.getSPIRVTypeID(ResType)) 1739 .addUse(TmpVec) 1740 .addImm(0) 1741 .constrainAllUses(TII, TRI, RBI); 1742 1743 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1744 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1745 1746 Result &= 1747 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1748 .addDef(Elt) 1749 .addUse(GR.getSPIRVTypeID(ResType)) 1750 .addUse(TmpVec) 1751 .addImm(i) 1752 .constrainAllUses(TII, TRI, RBI); 1753 1754 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1755 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1756 : ResVReg; 1757 1758 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1759 .addDef(Sum) 1760 .addUse(GR.getSPIRVTypeID(ResType)) 1761 .addUse(Res) 1762 .addUse(Elt) 1763 .constrainAllUses(TII, TRI, RBI); 1764 Res = Sum; 1765 } 1766 1767 return Result; 1768 } 1769 1770 template <bool Signed> 1771 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1772 const SPIRVType *ResType, 1773 MachineInstr &I) const { 1774 assert(I.getNumOperands() == 5); 1775 assert(I.getOperand(2).isReg()); 1776 assert(I.getOperand(3).isReg()); 1777 assert(I.getOperand(4).isReg()); 1778 MachineBasicBlock &BB = *I.getParent(); 1779 1780 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1781 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1782 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1783 .addDef(Dot) 1784 .addUse(GR.getSPIRVTypeID(ResType)) 1785 .addUse(I.getOperand(2).getReg()) 1786 .addUse(I.getOperand(3).getReg()) 1787 .constrainAllUses(TII, TRI, RBI); 1788 1789 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1790 .addDef(ResVReg) 1791 .addUse(GR.getSPIRVTypeID(ResType)) 1792 .addUse(Dot) 1793 .addUse(I.getOperand(4).getReg()) 1794 .constrainAllUses(TII, TRI, RBI); 1795 } 1796 1797 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1798 // extract the elements of the packed inputs, multiply them and add the result 1799 // to the accumulator. 1800 template <bool Signed> 1801 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1802 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1803 assert(I.getNumOperands() == 5); 1804 assert(I.getOperand(2).isReg()); 1805 assert(I.getOperand(3).isReg()); 1806 assert(I.getOperand(4).isReg()); 1807 MachineBasicBlock &BB = *I.getParent(); 1808 1809 bool Result = true; 1810 1811 // Acc = C 1812 Register Acc = I.getOperand(4).getReg(); 1813 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1814 auto ExtractOp = 1815 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1816 1817 // Extract the i8 element, multiply and add it to the accumulator 1818 for (unsigned i = 0; i < 4; i++) { 1819 // A[i] 1820 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1821 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1822 .addDef(AElt) 1823 .addUse(GR.getSPIRVTypeID(ResType)) 1824 .addUse(I.getOperand(2).getReg()) 1825 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1826 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1827 .constrainAllUses(TII, TRI, RBI); 1828 1829 // B[i] 1830 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1831 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1832 .addDef(BElt) 1833 .addUse(GR.getSPIRVTypeID(ResType)) 1834 .addUse(I.getOperand(3).getReg()) 1835 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1836 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1837 .constrainAllUses(TII, TRI, RBI); 1838 1839 // A[i] * B[i] 1840 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1841 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1842 .addDef(Mul) 1843 .addUse(GR.getSPIRVTypeID(ResType)) 1844 .addUse(AElt) 1845 .addUse(BElt) 1846 .constrainAllUses(TII, TRI, RBI); 1847 1848 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1849 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1850 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1851 .addDef(MaskMul) 1852 .addUse(GR.getSPIRVTypeID(ResType)) 1853 .addUse(Mul) 1854 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1855 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1856 .constrainAllUses(TII, TRI, RBI); 1857 1858 // Acc = Acc + A[i] * B[i] 1859 Register Sum = 1860 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1861 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1862 .addDef(Sum) 1863 .addUse(GR.getSPIRVTypeID(ResType)) 1864 .addUse(Acc) 1865 .addUse(MaskMul) 1866 .constrainAllUses(TII, TRI, RBI); 1867 1868 Acc = Sum; 1869 } 1870 1871 return Result; 1872 } 1873 1874 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1875 /// does not have a saturate builtin. 1876 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1877 const SPIRVType *ResType, 1878 MachineInstr &I) const { 1879 assert(I.getNumOperands() == 3); 1880 assert(I.getOperand(2).isReg()); 1881 MachineBasicBlock &BB = *I.getParent(); 1882 Register VZero = buildZerosValF(ResType, I); 1883 Register VOne = buildOnesValF(ResType, I); 1884 1885 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1886 .addDef(ResVReg) 1887 .addUse(GR.getSPIRVTypeID(ResType)) 1888 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1889 .addImm(GL::FClamp) 1890 .addUse(I.getOperand(2).getReg()) 1891 .addUse(VZero) 1892 .addUse(VOne) 1893 .constrainAllUses(TII, TRI, RBI); 1894 } 1895 1896 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1897 const SPIRVType *ResType, 1898 MachineInstr &I) const { 1899 assert(I.getNumOperands() == 3); 1900 assert(I.getOperand(2).isReg()); 1901 MachineBasicBlock &BB = *I.getParent(); 1902 Register InputRegister = I.getOperand(2).getReg(); 1903 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1904 auto &DL = I.getDebugLoc(); 1905 1906 if (!InputType) 1907 report_fatal_error("Input Type could not be determined."); 1908 1909 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1910 1911 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1912 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1913 1914 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1915 1916 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1917 Register SignReg = NeedsConversion 1918 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1919 : ResVReg; 1920 1921 bool Result = 1922 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1923 .addDef(SignReg) 1924 .addUse(GR.getSPIRVTypeID(InputType)) 1925 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1926 .addImm(SignOpcode) 1927 .addUse(InputRegister) 1928 .constrainAllUses(TII, TRI, RBI); 1929 1930 if (NeedsConversion) { 1931 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1932 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1933 .addDef(ResVReg) 1934 .addUse(GR.getSPIRVTypeID(ResType)) 1935 .addUse(SignReg) 1936 .constrainAllUses(TII, TRI, RBI); 1937 } 1938 1939 return Result; 1940 } 1941 1942 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 1943 const SPIRVType *ResType, 1944 MachineInstr &I, 1945 unsigned Opcode) const { 1946 MachineBasicBlock &BB = *I.getParent(); 1947 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1948 1949 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1950 .addDef(ResVReg) 1951 .addUse(GR.getSPIRVTypeID(ResType)) 1952 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 1953 IntTy, TII)); 1954 1955 for (unsigned J = 2; J < I.getNumOperands(); J++) { 1956 BMI.addUse(I.getOperand(J).getReg()); 1957 } 1958 1959 return BMI.constrainAllUses(TII, TRI, RBI); 1960 } 1961 1962 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 1963 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1964 1965 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1966 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 1967 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 1968 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 1969 SPIRV::OpGroupNonUniformBallot); 1970 1971 MachineBasicBlock &BB = *I.getParent(); 1972 Result &= 1973 BuildMI(BB, I, I.getDebugLoc(), 1974 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 1975 .addDef(ResVReg) 1976 .addUse(GR.getSPIRVTypeID(ResType)) 1977 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1978 .addImm(SPIRV::GroupOperation::Reduce) 1979 .addUse(BallotReg) 1980 .constrainAllUses(TII, TRI, RBI); 1981 1982 return Result; 1983 } 1984 1985 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1986 const SPIRVType *ResType, 1987 MachineInstr &I) const { 1988 MachineBasicBlock &BB = *I.getParent(); 1989 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1990 .addDef(ResVReg) 1991 .addUse(GR.getSPIRVTypeID(ResType)) 1992 .addUse(I.getOperand(1).getReg()) 1993 .constrainAllUses(TII, TRI, RBI); 1994 } 1995 1996 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1997 const SPIRVType *ResType, 1998 MachineInstr &I) const { 1999 // There is no way to implement `freeze` correctly without support on SPIR-V 2000 // standard side, but we may at least address a simple (static) case when 2001 // undef/poison value presence is obvious. The main benefit of even 2002 // incomplete `freeze` support is preventing of translation from crashing due 2003 // to lack of support on legalization and instruction selection steps. 2004 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2005 return false; 2006 Register OpReg = I.getOperand(1).getReg(); 2007 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2008 Register Reg; 2009 switch (Def->getOpcode()) { 2010 case SPIRV::ASSIGN_TYPE: 2011 if (MachineInstr *AssignToDef = 2012 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2013 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2014 Reg = Def->getOperand(2).getReg(); 2015 } 2016 break; 2017 case SPIRV::OpUndef: 2018 Reg = Def->getOperand(1).getReg(); 2019 break; 2020 } 2021 unsigned DestOpCode; 2022 if (Reg.isValid()) { 2023 DestOpCode = SPIRV::OpConstantNull; 2024 } else { 2025 DestOpCode = TargetOpcode::COPY; 2026 Reg = OpReg; 2027 } 2028 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2029 .addDef(I.getOperand(0).getReg()) 2030 .addUse(Reg) 2031 .constrainAllUses(TII, TRI, RBI); 2032 } 2033 return false; 2034 } 2035 2036 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2037 const SPIRVType *ResType) { 2038 Register OpReg = ResType->getOperand(2).getReg(); 2039 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2040 if (!OpDef) 2041 return 0; 2042 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2043 OpDef->getOperand(1).isReg()) { 2044 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2045 OpDef = RefDef; 2046 } 2047 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2048 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2049 : 0; 2050 return N; 2051 } 2052 2053 // Return true if the type represents a constant register 2054 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2055 SmallPtrSet<SPIRVType *, 4> &Visited) { 2056 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2057 OpDef->getOperand(1).isReg()) { 2058 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2059 OpDef = RefDef; 2060 } 2061 2062 if (Visited.contains(OpDef)) 2063 return true; 2064 Visited.insert(OpDef); 2065 2066 unsigned Opcode = OpDef->getOpcode(); 2067 switch (Opcode) { 2068 case TargetOpcode::G_CONSTANT: 2069 case TargetOpcode::G_FCONSTANT: 2070 return true; 2071 case TargetOpcode::G_INTRINSIC: 2072 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2073 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2074 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2075 Intrinsic::spv_const_composite; 2076 case TargetOpcode::G_BUILD_VECTOR: 2077 case TargetOpcode::G_SPLAT_VECTOR: { 2078 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2079 i++) { 2080 SPIRVType *OpNestedDef = 2081 OpDef->getOperand(i).isReg() 2082 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2083 : nullptr; 2084 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2085 return false; 2086 } 2087 return true; 2088 } 2089 } 2090 return false; 2091 } 2092 2093 // Return true if the virtual register represents a constant 2094 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2095 SmallPtrSet<SPIRVType *, 4> Visited; 2096 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2097 return isConstReg(MRI, OpDef, Visited); 2098 return false; 2099 } 2100 2101 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2102 const SPIRVType *ResType, 2103 MachineInstr &I) const { 2104 unsigned N = 0; 2105 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2106 N = GR.getScalarOrVectorComponentCount(ResType); 2107 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2108 N = getArrayComponentCount(MRI, ResType); 2109 else 2110 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2111 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2112 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2113 2114 // check if we may construct a constant vector 2115 bool IsConst = true; 2116 for (unsigned i = I.getNumExplicitDefs(); 2117 i < I.getNumExplicitOperands() && IsConst; ++i) 2118 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2119 IsConst = false; 2120 2121 if (!IsConst && N < 2) 2122 report_fatal_error( 2123 "There must be at least two constituent operands in a vector"); 2124 2125 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2126 TII.get(IsConst ? SPIRV::OpConstantComposite 2127 : SPIRV::OpCompositeConstruct)) 2128 .addDef(ResVReg) 2129 .addUse(GR.getSPIRVTypeID(ResType)); 2130 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2131 MIB.addUse(I.getOperand(i).getReg()); 2132 return MIB.constrainAllUses(TII, TRI, RBI); 2133 } 2134 2135 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2136 const SPIRVType *ResType, 2137 MachineInstr &I) const { 2138 unsigned N = 0; 2139 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2140 N = GR.getScalarOrVectorComponentCount(ResType); 2141 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2142 N = getArrayComponentCount(MRI, ResType); 2143 else 2144 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2145 2146 unsigned OpIdx = I.getNumExplicitDefs(); 2147 if (!I.getOperand(OpIdx).isReg()) 2148 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2149 2150 // check if we may construct a constant vector 2151 Register OpReg = I.getOperand(OpIdx).getReg(); 2152 bool IsConst = isConstReg(MRI, OpReg); 2153 2154 if (!IsConst && N < 2) 2155 report_fatal_error( 2156 "There must be at least two constituent operands in a vector"); 2157 2158 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2159 TII.get(IsConst ? SPIRV::OpConstantComposite 2160 : SPIRV::OpCompositeConstruct)) 2161 .addDef(ResVReg) 2162 .addUse(GR.getSPIRVTypeID(ResType)); 2163 for (unsigned i = 0; i < N; ++i) 2164 MIB.addUse(OpReg); 2165 return MIB.constrainAllUses(TII, TRI, RBI); 2166 } 2167 2168 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2169 const SPIRVType *ResType, 2170 MachineInstr &I) const { 2171 2172 unsigned Opcode; 2173 2174 if (STI.canUseExtension( 2175 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2176 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2177 Opcode = SPIRV::OpDemoteToHelperInvocation; 2178 } else { 2179 Opcode = SPIRV::OpKill; 2180 // OpKill must be the last operation of any basic block. 2181 MachineInstr *NextI = I.getNextNode(); 2182 NextI->removeFromParent(); 2183 } 2184 2185 MachineBasicBlock &BB = *I.getParent(); 2186 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2187 .constrainAllUses(TII, TRI, RBI); 2188 } 2189 2190 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2191 const SPIRVType *ResType, 2192 unsigned CmpOpc, 2193 MachineInstr &I) const { 2194 Register Cmp0 = I.getOperand(2).getReg(); 2195 Register Cmp1 = I.getOperand(3).getReg(); 2196 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2197 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2198 "CMP operands should have the same type"); 2199 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2200 .addDef(ResVReg) 2201 .addUse(GR.getSPIRVTypeID(ResType)) 2202 .addUse(Cmp0) 2203 .addUse(Cmp1) 2204 .constrainAllUses(TII, TRI, RBI); 2205 } 2206 2207 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2208 const SPIRVType *ResType, 2209 MachineInstr &I) const { 2210 auto Pred = I.getOperand(1).getPredicate(); 2211 unsigned CmpOpc; 2212 2213 Register CmpOperand = I.getOperand(2).getReg(); 2214 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2215 CmpOpc = getPtrCmpOpcode(Pred); 2216 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2217 CmpOpc = getBoolCmpOpcode(Pred); 2218 else 2219 CmpOpc = getICmpOpcode(Pred); 2220 return selectCmp(ResVReg, ResType, CmpOpc, I); 2221 } 2222 2223 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2224 const MachineInstr &I, 2225 int OpIdx) const { 2226 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2227 "Expected G_FCONSTANT"); 2228 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2229 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2230 } 2231 2232 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2233 const MachineInstr &I, 2234 int OpIdx) const { 2235 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2236 "Expected G_CONSTANT"); 2237 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2238 } 2239 2240 std::pair<Register, bool> 2241 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2242 const SPIRVType *ResType) const { 2243 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2244 const SPIRVType *SpvI32Ty = 2245 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2246 // Find a constant in DT or build a new one. 2247 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2248 Register NewReg = GR.find(ConstInt, GR.CurMF); 2249 bool Result = true; 2250 if (!NewReg.isValid()) { 2251 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2252 GR.add(ConstInt, GR.CurMF, NewReg); 2253 MachineInstr *MI; 2254 MachineBasicBlock &BB = *I.getParent(); 2255 if (Val == 0) { 2256 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2257 .addDef(NewReg) 2258 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2259 } else { 2260 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2261 .addDef(NewReg) 2262 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2263 .addImm(APInt(32, Val).getZExtValue()); 2264 } 2265 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2266 } 2267 return {NewReg, Result}; 2268 } 2269 2270 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2271 const SPIRVType *ResType, 2272 MachineInstr &I) const { 2273 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2274 return selectCmp(ResVReg, ResType, CmpOp, I); 2275 } 2276 2277 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2278 MachineInstr &I) const { 2279 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2280 bool ZeroAsNull = STI.isOpenCLEnv(); 2281 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2282 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2283 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2284 } 2285 2286 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2287 MachineInstr &I) const { 2288 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2289 bool ZeroAsNull = STI.isOpenCLEnv(); 2290 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2291 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2292 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2293 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2294 } 2295 2296 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2297 MachineInstr &I) const { 2298 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2299 bool ZeroAsNull = STI.isOpenCLEnv(); 2300 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2301 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2302 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2303 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2304 } 2305 2306 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2307 const SPIRVType *ResType, 2308 MachineInstr &I) const { 2309 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2310 APInt One = 2311 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2312 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2313 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2314 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2315 } 2316 2317 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2318 const SPIRVType *ResType, 2319 MachineInstr &I, 2320 bool IsSigned) const { 2321 // To extend a bool, we need to use OpSelect between constants. 2322 Register ZeroReg = buildZerosVal(ResType, I); 2323 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2324 bool IsScalarBool = 2325 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2326 unsigned Opcode = 2327 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2328 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2329 .addDef(ResVReg) 2330 .addUse(GR.getSPIRVTypeID(ResType)) 2331 .addUse(I.getOperand(1).getReg()) 2332 .addUse(OneReg) 2333 .addUse(ZeroReg) 2334 .constrainAllUses(TII, TRI, RBI); 2335 } 2336 2337 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2338 const SPIRVType *ResType, 2339 MachineInstr &I, bool IsSigned, 2340 unsigned Opcode) const { 2341 Register SrcReg = I.getOperand(1).getReg(); 2342 // We can convert bool value directly to float type without OpConvert*ToF, 2343 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2344 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2345 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2346 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2347 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2348 const unsigned NumElts = ResType->getOperand(2).getImm(); 2349 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2350 } 2351 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2352 selectSelect(SrcReg, TmpType, I, false); 2353 } 2354 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2355 } 2356 2357 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2358 const SPIRVType *ResType, 2359 MachineInstr &I, bool IsSigned) const { 2360 Register SrcReg = I.getOperand(1).getReg(); 2361 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2362 return selectSelect(ResVReg, ResType, I, IsSigned); 2363 2364 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2365 if (SrcType == ResType) { 2366 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2367 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2368 if (DstRC != SrcRC && SrcRC) 2369 MRI->setRegClass(ResVReg, SrcRC); 2370 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2371 TII.get(TargetOpcode::COPY)) 2372 .addDef(ResVReg) 2373 .addUse(SrcReg) 2374 .constrainAllUses(TII, TRI, RBI); 2375 } 2376 2377 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2378 return selectUnOp(ResVReg, ResType, I, Opcode); 2379 } 2380 2381 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2382 Register ResVReg, 2383 MachineInstr &I, 2384 const SPIRVType *IntTy, 2385 const SPIRVType *BoolTy) const { 2386 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2387 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2388 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2389 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2390 Register Zero = buildZerosVal(IntTy, I); 2391 Register One = buildOnesVal(false, IntTy, I); 2392 MachineBasicBlock &BB = *I.getParent(); 2393 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2394 .addDef(BitIntReg) 2395 .addUse(GR.getSPIRVTypeID(IntTy)) 2396 .addUse(IntReg) 2397 .addUse(One) 2398 .constrainAllUses(TII, TRI, RBI); 2399 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2400 .addDef(ResVReg) 2401 .addUse(GR.getSPIRVTypeID(BoolTy)) 2402 .addUse(BitIntReg) 2403 .addUse(Zero) 2404 .constrainAllUses(TII, TRI, RBI); 2405 } 2406 2407 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2408 const SPIRVType *ResType, 2409 MachineInstr &I) const { 2410 Register IntReg = I.getOperand(1).getReg(); 2411 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2412 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2413 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2414 if (ArgType == ResType) { 2415 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2416 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2417 if (DstRC != SrcRC && SrcRC) 2418 MRI->setRegClass(ResVReg, SrcRC); 2419 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2420 TII.get(TargetOpcode::COPY)) 2421 .addDef(ResVReg) 2422 .addUse(IntReg) 2423 .constrainAllUses(TII, TRI, RBI); 2424 } 2425 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2426 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2427 return selectUnOp(ResVReg, ResType, I, Opcode); 2428 } 2429 2430 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2431 const SPIRVType *ResType, 2432 const APInt &Imm, 2433 MachineInstr &I) const { 2434 unsigned TyOpcode = ResType->getOpcode(); 2435 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2436 MachineBasicBlock &BB = *I.getParent(); 2437 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2438 Imm.isZero()) 2439 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2440 .addDef(ResVReg) 2441 .addUse(GR.getSPIRVTypeID(ResType)) 2442 .constrainAllUses(TII, TRI, RBI); 2443 if (TyOpcode == SPIRV::OpTypeInt) { 2444 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2445 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2446 if (Reg == ResVReg) 2447 return true; 2448 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2449 .addDef(ResVReg) 2450 .addUse(Reg) 2451 .constrainAllUses(TII, TRI, RBI); 2452 } 2453 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2454 .addDef(ResVReg) 2455 .addUse(GR.getSPIRVTypeID(ResType)); 2456 // <=32-bit integers should be caught by the sdag pattern. 2457 assert(Imm.getBitWidth() > 32); 2458 addNumImm(Imm, MIB); 2459 return MIB.constrainAllUses(TII, TRI, RBI); 2460 } 2461 2462 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2463 const SPIRVType *ResType, 2464 MachineInstr &I) const { 2465 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2466 .addDef(ResVReg) 2467 .addUse(GR.getSPIRVTypeID(ResType)) 2468 .constrainAllUses(TII, TRI, RBI); 2469 } 2470 2471 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2472 assert(MO.isReg()); 2473 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2474 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2475 assert(TypeInst->getOperand(1).isReg()); 2476 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2477 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2478 } 2479 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2480 } 2481 2482 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2483 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2484 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2485 return TypeInst->getOperand(2).getImm(); 2486 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2487 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2488 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2489 } 2490 2491 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2492 const SPIRVType *ResType, 2493 MachineInstr &I) const { 2494 MachineBasicBlock &BB = *I.getParent(); 2495 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2496 .addDef(ResVReg) 2497 .addUse(GR.getSPIRVTypeID(ResType)) 2498 // object to insert 2499 .addUse(I.getOperand(3).getReg()) 2500 // composite to insert into 2501 .addUse(I.getOperand(2).getReg()); 2502 for (unsigned i = 4; i < I.getNumOperands(); i++) 2503 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2504 return MIB.constrainAllUses(TII, TRI, RBI); 2505 } 2506 2507 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2508 const SPIRVType *ResType, 2509 MachineInstr &I) const { 2510 MachineBasicBlock &BB = *I.getParent(); 2511 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2512 .addDef(ResVReg) 2513 .addUse(GR.getSPIRVTypeID(ResType)) 2514 .addUse(I.getOperand(2).getReg()); 2515 for (unsigned i = 3; i < I.getNumOperands(); i++) 2516 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2517 return MIB.constrainAllUses(TII, TRI, RBI); 2518 } 2519 2520 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2521 const SPIRVType *ResType, 2522 MachineInstr &I) const { 2523 if (isImm(I.getOperand(4), MRI)) 2524 return selectInsertVal(ResVReg, ResType, I); 2525 MachineBasicBlock &BB = *I.getParent(); 2526 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2527 .addDef(ResVReg) 2528 .addUse(GR.getSPIRVTypeID(ResType)) 2529 .addUse(I.getOperand(2).getReg()) 2530 .addUse(I.getOperand(3).getReg()) 2531 .addUse(I.getOperand(4).getReg()) 2532 .constrainAllUses(TII, TRI, RBI); 2533 } 2534 2535 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2536 const SPIRVType *ResType, 2537 MachineInstr &I) const { 2538 if (isImm(I.getOperand(3), MRI)) 2539 return selectExtractVal(ResVReg, ResType, I); 2540 MachineBasicBlock &BB = *I.getParent(); 2541 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2542 .addDef(ResVReg) 2543 .addUse(GR.getSPIRVTypeID(ResType)) 2544 .addUse(I.getOperand(2).getReg()) 2545 .addUse(I.getOperand(3).getReg()) 2546 .constrainAllUses(TII, TRI, RBI); 2547 } 2548 2549 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2550 const SPIRVType *ResType, 2551 MachineInstr &I) const { 2552 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2553 2554 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2555 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2556 // we have to use Op[InBounds]AccessChain. 2557 const unsigned Opcode = STI.isVulkanEnv() 2558 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2559 : SPIRV::OpAccessChain) 2560 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2561 : SPIRV::OpPtrAccessChain); 2562 2563 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2564 .addDef(ResVReg) 2565 .addUse(GR.getSPIRVTypeID(ResType)) 2566 // Object to get a pointer to. 2567 .addUse(I.getOperand(3).getReg()); 2568 // Adding indices. 2569 const unsigned StartingIndex = 2570 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2571 ? 5 2572 : 4; 2573 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2574 Res.addUse(I.getOperand(i).getReg()); 2575 return Res.constrainAllUses(TII, TRI, RBI); 2576 } 2577 2578 // Maybe wrap a value into OpSpecConstantOp 2579 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2580 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2581 bool Result = true; 2582 unsigned Lim = I.getNumExplicitOperands(); 2583 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2584 Register OpReg = I.getOperand(i).getReg(); 2585 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2586 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2587 SmallPtrSet<SPIRVType *, 4> Visited; 2588 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2589 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2590 GR.isAggregateType(OpType)) { 2591 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2592 // by selectAddrSpaceCast() 2593 CompositeArgs.push_back(OpReg); 2594 continue; 2595 } 2596 MachineFunction *MF = I.getMF(); 2597 Register WrapReg = GR.find(OpDefine, MF); 2598 if (WrapReg.isValid()) { 2599 CompositeArgs.push_back(WrapReg); 2600 continue; 2601 } 2602 // Create a new register for the wrapper 2603 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2604 GR.add(OpDefine, MF, WrapReg); 2605 CompositeArgs.push_back(WrapReg); 2606 // Decorate the wrapper register and generate a new instruction 2607 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2608 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2609 MachineBasicBlock &BB = *I.getParent(); 2610 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2611 .addDef(WrapReg) 2612 .addUse(GR.getSPIRVTypeID(OpType)) 2613 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2614 .addUse(OpReg) 2615 .constrainAllUses(TII, TRI, RBI); 2616 if (!Result) 2617 break; 2618 } 2619 return Result; 2620 } 2621 2622 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2623 const SPIRVType *ResType, 2624 MachineInstr &I) const { 2625 MachineBasicBlock &BB = *I.getParent(); 2626 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2627 switch (IID) { 2628 case Intrinsic::spv_load: 2629 return selectLoad(ResVReg, ResType, I); 2630 case Intrinsic::spv_store: 2631 return selectStore(I); 2632 case Intrinsic::spv_extractv: 2633 return selectExtractVal(ResVReg, ResType, I); 2634 case Intrinsic::spv_insertv: 2635 return selectInsertVal(ResVReg, ResType, I); 2636 case Intrinsic::spv_extractelt: 2637 return selectExtractElt(ResVReg, ResType, I); 2638 case Intrinsic::spv_insertelt: 2639 return selectInsertElt(ResVReg, ResType, I); 2640 case Intrinsic::spv_gep: 2641 return selectGEP(ResVReg, ResType, I); 2642 case Intrinsic::spv_unref_global: 2643 case Intrinsic::spv_init_global: { 2644 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2645 MachineInstr *Init = I.getNumExplicitOperands() > 2 2646 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2647 : nullptr; 2648 assert(MI); 2649 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2650 } 2651 case Intrinsic::spv_undef: { 2652 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2653 .addDef(ResVReg) 2654 .addUse(GR.getSPIRVTypeID(ResType)); 2655 return MIB.constrainAllUses(TII, TRI, RBI); 2656 } 2657 case Intrinsic::spv_const_composite: { 2658 // If no values are attached, the composite is null constant. 2659 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2660 // Select a proper instruction. 2661 unsigned Opcode = SPIRV::OpConstantNull; 2662 SmallVector<Register> CompositeArgs; 2663 if (!IsNull) { 2664 Opcode = SPIRV::OpConstantComposite; 2665 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2666 return false; 2667 } 2668 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2669 .addDef(ResVReg) 2670 .addUse(GR.getSPIRVTypeID(ResType)); 2671 // skip type MD node we already used when generated assign.type for this 2672 if (!IsNull) { 2673 for (Register OpReg : CompositeArgs) 2674 MIB.addUse(OpReg); 2675 } 2676 return MIB.constrainAllUses(TII, TRI, RBI); 2677 } 2678 case Intrinsic::spv_assign_name: { 2679 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2680 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2681 for (unsigned i = I.getNumExplicitDefs() + 2; 2682 i < I.getNumExplicitOperands(); ++i) { 2683 MIB.addImm(I.getOperand(i).getImm()); 2684 } 2685 return MIB.constrainAllUses(TII, TRI, RBI); 2686 } 2687 case Intrinsic::spv_switch: { 2688 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2689 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2690 if (I.getOperand(i).isReg()) 2691 MIB.addReg(I.getOperand(i).getReg()); 2692 else if (I.getOperand(i).isCImm()) 2693 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2694 else if (I.getOperand(i).isMBB()) 2695 MIB.addMBB(I.getOperand(i).getMBB()); 2696 else 2697 llvm_unreachable("Unexpected OpSwitch operand"); 2698 } 2699 return MIB.constrainAllUses(TII, TRI, RBI); 2700 } 2701 case Intrinsic::spv_loop_merge: 2702 case Intrinsic::spv_selection_merge: { 2703 const auto Opcode = IID == Intrinsic::spv_selection_merge 2704 ? SPIRV::OpSelectionMerge 2705 : SPIRV::OpLoopMerge; 2706 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2707 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2708 assert(I.getOperand(i).isMBB()); 2709 MIB.addMBB(I.getOperand(i).getMBB()); 2710 } 2711 MIB.addImm(SPIRV::SelectionControl::None); 2712 return MIB.constrainAllUses(TII, TRI, RBI); 2713 } 2714 case Intrinsic::spv_cmpxchg: 2715 return selectAtomicCmpXchg(ResVReg, ResType, I); 2716 case Intrinsic::spv_unreachable: 2717 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2718 .constrainAllUses(TII, TRI, RBI); 2719 case Intrinsic::spv_alloca: 2720 return selectFrameIndex(ResVReg, ResType, I); 2721 case Intrinsic::spv_alloca_array: 2722 return selectAllocaArray(ResVReg, ResType, I); 2723 case Intrinsic::spv_assume: 2724 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2725 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2726 .addUse(I.getOperand(1).getReg()) 2727 .constrainAllUses(TII, TRI, RBI); 2728 break; 2729 case Intrinsic::spv_expect: 2730 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2731 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2732 .addDef(ResVReg) 2733 .addUse(GR.getSPIRVTypeID(ResType)) 2734 .addUse(I.getOperand(2).getReg()) 2735 .addUse(I.getOperand(3).getReg()) 2736 .constrainAllUses(TII, TRI, RBI); 2737 break; 2738 case Intrinsic::arithmetic_fence: 2739 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2740 return BuildMI(BB, I, I.getDebugLoc(), 2741 TII.get(SPIRV::OpArithmeticFenceEXT)) 2742 .addDef(ResVReg) 2743 .addUse(GR.getSPIRVTypeID(ResType)) 2744 .addUse(I.getOperand(2).getReg()) 2745 .constrainAllUses(TII, TRI, RBI); 2746 else 2747 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), 2748 ResVReg) 2749 .addUse(I.getOperand(2).getReg()) 2750 .constrainAllUses(TII, TRI, RBI); 2751 break; 2752 case Intrinsic::spv_thread_id: 2753 return selectSpvThreadId(ResVReg, ResType, I); 2754 case Intrinsic::spv_fdot: 2755 return selectFloatDot(ResVReg, ResType, I); 2756 case Intrinsic::spv_udot: 2757 case Intrinsic::spv_sdot: 2758 return selectIntegerDot(ResVReg, ResType, I); 2759 case Intrinsic::spv_dot4add_i8packed: 2760 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2761 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2762 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2763 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2764 case Intrinsic::spv_dot4add_u8packed: 2765 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2766 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2767 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2768 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2769 case Intrinsic::spv_all: 2770 return selectAll(ResVReg, ResType, I); 2771 case Intrinsic::spv_any: 2772 return selectAny(ResVReg, ResType, I); 2773 case Intrinsic::spv_cross: 2774 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2775 case Intrinsic::spv_lerp: 2776 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2777 case Intrinsic::spv_length: 2778 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2779 case Intrinsic::spv_degrees: 2780 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2781 case Intrinsic::spv_frac: 2782 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2783 case Intrinsic::spv_normalize: 2784 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2785 case Intrinsic::spv_rsqrt: 2786 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2787 case Intrinsic::spv_sign: 2788 return selectSign(ResVReg, ResType, I); 2789 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2790 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2791 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2792 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2793 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2794 bool Result = true; 2795 auto MemSemConstant = 2796 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2797 Register MemSemReg = MemSemConstant.first; 2798 Result &= MemSemConstant.second; 2799 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2800 Register ScopeReg = ScopeConstant.first; 2801 Result &= ScopeConstant.second; 2802 MachineBasicBlock &BB = *I.getParent(); 2803 return Result && 2804 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2805 .addUse(ScopeReg) 2806 .addUse(ScopeReg) 2807 .addUse(MemSemReg) 2808 .constrainAllUses(TII, TRI, RBI); 2809 } 2810 case Intrinsic::spv_lifetime_start: 2811 case Intrinsic::spv_lifetime_end: { 2812 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2813 : SPIRV::OpLifetimeStop; 2814 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2815 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2816 if (Size == -1) 2817 Size = 0; 2818 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2819 .addUse(PtrReg) 2820 .addImm(Size) 2821 .constrainAllUses(TII, TRI, RBI); 2822 } 2823 case Intrinsic::spv_saturate: 2824 return selectSaturate(ResVReg, ResType, I); 2825 case Intrinsic::spv_nclamp: 2826 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2827 case Intrinsic::spv_uclamp: 2828 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2829 case Intrinsic::spv_sclamp: 2830 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2831 case Intrinsic::spv_wave_active_countbits: 2832 return selectWaveActiveCountBits(ResVReg, ResType, I); 2833 case Intrinsic::spv_wave_any: 2834 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 2835 case Intrinsic::spv_wave_is_first_lane: 2836 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 2837 case Intrinsic::spv_wave_readlane: 2838 return selectWaveOpInst(ResVReg, ResType, I, 2839 SPIRV::OpGroupNonUniformShuffle); 2840 case Intrinsic::spv_step: 2841 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2842 case Intrinsic::spv_radians: 2843 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2844 // Discard intrinsics which we do not expect to actually represent code after 2845 // lowering or intrinsics which are not implemented but should not crash when 2846 // found in a customer's LLVM IR input. 2847 case Intrinsic::instrprof_increment: 2848 case Intrinsic::instrprof_increment_step: 2849 case Intrinsic::instrprof_value_profile: 2850 break; 2851 // Discard internal intrinsics. 2852 case Intrinsic::spv_value_md: 2853 break; 2854 case Intrinsic::spv_handle_fromBinding: { 2855 return selectHandleFromBinding(ResVReg, ResType, I); 2856 } 2857 case Intrinsic::spv_typedBufferStore: { 2858 selectImageWriteIntrinsic(I); 2859 return true; 2860 } 2861 case Intrinsic::spv_typedBufferLoad: { 2862 selectReadImageIntrinsic(ResVReg, ResType, I); 2863 return true; 2864 } 2865 case Intrinsic::spv_discard: { 2866 return selectDiscard(ResVReg, ResType, I); 2867 } 2868 default: { 2869 std::string DiagMsg; 2870 raw_string_ostream OS(DiagMsg); 2871 I.print(OS); 2872 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2873 report_fatal_error(DiagMsg.c_str(), false); 2874 } 2875 } 2876 return true; 2877 } 2878 2879 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2880 const SPIRVType *ResType, 2881 MachineInstr &I) const { 2882 2883 uint32_t Set = foldImm(I.getOperand(2), MRI); 2884 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2885 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2886 Register IndexReg = I.getOperand(5).getReg(); 2887 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2888 2889 MachineIRBuilder MIRBuilder(I); 2890 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2891 IndexReg, IsNonUniform, MIRBuilder); 2892 2893 if (IsNonUniform) 2894 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2895 2896 // TODO: For now we assume the resource is an image, which needs to be 2897 // loaded to get the handle. That will not be true for storage buffers. 2898 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2899 .addDef(ResVReg) 2900 .addUse(GR.getSPIRVTypeID(ResType)) 2901 .addUse(VarReg) 2902 .constrainAllUses(TII, TRI, RBI); 2903 } 2904 2905 void SPIRVInstructionSelector::selectReadImageIntrinsic( 2906 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2907 2908 // If the load of the image is in a different basic block, then 2909 // this will generate invalid code. A proper solution is to move 2910 // the OpLoad from selectHandleFromBinding here. However, to do 2911 // that we will need to change the return type of the intrinsic. 2912 // We will do that when we can, but for now trying to move forward with other 2913 // issues. 2914 Register ImageReg = I.getOperand(2).getReg(); 2915 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 2916 "The image must be loaded in the same basic block as its use."); 2917 2918 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2919 if (ResultSize == 4) { 2920 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 2921 .addDef(ResVReg) 2922 .addUse(GR.getSPIRVTypeID(ResType)) 2923 .addUse(ImageReg) 2924 .addUse(I.getOperand(3).getReg()); 2925 return; 2926 } 2927 2928 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 2929 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 2930 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 2931 .addDef(ReadReg) 2932 .addUse(GR.getSPIRVTypeID(ReadType)) 2933 .addUse(ImageReg) 2934 .addUse(I.getOperand(3).getReg()); 2935 2936 if (ResultSize == 1) { 2937 BuildMI(*I.getParent(), I, I.getDebugLoc(), 2938 TII.get(SPIRV::OpCompositeExtract)) 2939 .addDef(ResVReg) 2940 .addUse(GR.getSPIRVTypeID(ResType)) 2941 .addUse(ReadReg) 2942 .addImm(0); 2943 return; 2944 } 2945 extractSubvector(ResVReg, ResType, ReadReg, I); 2946 } 2947 2948 void SPIRVInstructionSelector::extractSubvector( 2949 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 2950 MachineInstr &InsertionPoint) const { 2951 SPIRVType *InputType = GR.getResultType(ReadReg); 2952 [[maybe_unused]] uint64_t InputSize = 2953 GR.getScalarOrVectorComponentCount(InputType); 2954 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2955 assert(InputSize > 1 && "The input must be a vector."); 2956 assert(ResultSize > 1 && "The result must be a vector."); 2957 assert(ResultSize < InputSize && 2958 "Cannot extract more element than there are in the input."); 2959 SmallVector<Register> ComponentRegisters; 2960 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 2961 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 2962 for (uint64_t I = 0; I < ResultSize; I++) { 2963 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 2964 BuildMI(*InsertionPoint.getParent(), InsertionPoint, 2965 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2966 .addDef(ComponentReg) 2967 .addUse(ScalarType->getOperand(0).getReg()) 2968 .addUse(ReadReg) 2969 .addImm(I); 2970 ComponentRegisters.emplace_back(ComponentReg); 2971 } 2972 2973 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 2974 InsertionPoint.getDebugLoc(), 2975 TII.get(SPIRV::OpCompositeConstruct)) 2976 .addDef(ResVReg) 2977 .addUse(GR.getSPIRVTypeID(ResType)); 2978 2979 for (Register ComponentReg : ComponentRegisters) 2980 MIB.addUse(ComponentReg); 2981 } 2982 2983 void SPIRVInstructionSelector::selectImageWriteIntrinsic( 2984 MachineInstr &I) const { 2985 // If the load of the image is in a different basic block, then 2986 // this will generate invalid code. A proper solution is to move 2987 // the OpLoad from selectHandleFromBinding here. However, to do 2988 // that we will need to change the return type of the intrinsic. 2989 // We will do that when we can, but for now trying to move forward with other 2990 // issues. 2991 Register ImageReg = I.getOperand(1).getReg(); 2992 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 2993 "The image must be loaded in the same basic block as its use."); 2994 Register CoordinateReg = I.getOperand(2).getReg(); 2995 Register DataReg = I.getOperand(3).getReg(); 2996 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 2997 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 2998 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite)) 2999 .addUse(ImageReg) 3000 .addUse(CoordinateReg) 3001 .addUse(DataReg); 3002 } 3003 3004 Register SPIRVInstructionSelector::buildPointerToResource( 3005 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3006 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3007 MachineIRBuilder MIRBuilder) const { 3008 if (ArraySize == 1) 3009 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3010 MIRBuilder); 3011 3012 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3013 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3014 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3015 VarType, Set, Binding, MIRBuilder); 3016 3017 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3018 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3019 3020 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3021 if (IsNonUniform) { 3022 // It is unclear which value needs to be marked an non-uniform, so both 3023 // the index and the access changed are decorated as non-uniform. 3024 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3025 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3026 } 3027 3028 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3029 .addDef(AcReg) 3030 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3031 .addUse(VarReg) 3032 .addUse(IndexReg); 3033 3034 return AcReg; 3035 } 3036 3037 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3038 const SPIRVType *ResType, 3039 MachineInstr &I, 3040 bool IsSigned) const { 3041 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3042 // zero or sign extend 3043 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3044 bool Result = 3045 selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, Opcode); 3046 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3047 } 3048 3049 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3050 const SPIRVType *ResType, 3051 MachineInstr &I, 3052 Register SrcReg, 3053 bool IsSigned) const { 3054 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3055 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3056 .addDef(ResVReg) 3057 .addUse(GR.getSPIRVTypeID(ResType)) 3058 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3059 .addImm(Opcode) 3060 .addUse(SrcReg) 3061 .constrainAllUses(TII, TRI, RBI); 3062 } 3063 3064 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3065 const SPIRVType *ResType, 3066 MachineInstr &I, 3067 bool IsSigned) const { 3068 Register OpReg = I.getOperand(2).getReg(); 3069 // 1. split our int64 into 2 pieces using a bitcast 3070 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3071 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3072 MachineIRBuilder MIRBuilder(I); 3073 SPIRVType *postCastT = 3074 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3075 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3076 bool Result = 3077 selectOpWithSrcs(bitcastReg, postCastT, I, {OpReg}, SPIRV::OpBitcast); 3078 3079 // 2. call firstbithigh 3080 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3081 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3082 3083 // 3. split result vector into high bits and low bits 3084 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3085 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3086 3087 bool ZeroAsNull = STI.isOpenCLEnv(); 3088 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3089 if (isScalarRes) { 3090 // if scalar do a vector extract 3091 Result &= selectOpWithSrcs( 3092 HighReg, ResType, I, 3093 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3094 SPIRV::OpVectorExtractDynamic); 3095 Result &= selectOpWithSrcs( 3096 LowReg, ResType, I, 3097 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3098 SPIRV::OpVectorExtractDynamic); 3099 } else { // vector case do a shufflevector 3100 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3101 TII.get(SPIRV::OpVectorShuffle)) 3102 .addDef(HighReg) 3103 .addUse(GR.getSPIRVTypeID(ResType)) 3104 .addUse(FBHReg) 3105 .addUse(FBHReg); 3106 // ^^ this vector will not be selected from; could be empty 3107 unsigned j; 3108 for (j = 0; j < count * 2; j += 2) { 3109 MIB.addImm(j); 3110 } 3111 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3112 3113 // get low bits 3114 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3115 TII.get(SPIRV::OpVectorShuffle)) 3116 .addDef(LowReg) 3117 .addUse(GR.getSPIRVTypeID(ResType)) 3118 .addUse(FBHReg) 3119 .addUse(FBHReg); 3120 // ^^ this vector will not be selected from; could be empty 3121 for (j = 1; j < count * 2; j += 2) { 3122 MIB.addImm(j); 3123 } 3124 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3125 } 3126 3127 // 4. check if result of each top 32 bits is == -1 3128 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3129 Register NegOneReg; 3130 Register Reg0; 3131 Register Reg32; 3132 unsigned selectOp; 3133 unsigned addOp; 3134 if (isScalarRes) { 3135 NegOneReg = 3136 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3137 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3138 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3139 selectOp = SPIRV::OpSelectSISCond; 3140 addOp = SPIRV::OpIAddS; 3141 } else { 3142 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3143 NegOneReg = 3144 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3145 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3146 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3147 selectOp = SPIRV::OpSelectVIVCond; 3148 addOp = SPIRV::OpIAddV; 3149 } 3150 3151 // check if the high bits are == -1; true if -1 3152 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3153 Result &= selectOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3154 SPIRV::OpIEqual); 3155 3156 // Select low bits if true in BReg, otherwise high bits 3157 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3158 Result &= 3159 selectOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, selectOp); 3160 3161 // Add 32 for high bits, 0 for low bits 3162 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3163 Result &= selectOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3164 3165 return Result && 3166 selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3167 } 3168 3169 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3170 const SPIRVType *ResType, 3171 MachineInstr &I, 3172 bool IsSigned) const { 3173 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3174 Register OpReg = I.getOperand(2).getReg(); 3175 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3176 3177 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3178 case 16: 3179 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3180 case 32: 3181 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3182 case 64: 3183 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3184 default: 3185 report_fatal_error( 3186 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3187 } 3188 } 3189 3190 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3191 const SPIRVType *ResType, 3192 MachineInstr &I) const { 3193 // there was an allocation size parameter to the allocation instruction 3194 // that is not 1 3195 MachineBasicBlock &BB = *I.getParent(); 3196 return BuildMI(BB, I, I.getDebugLoc(), 3197 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3198 .addDef(ResVReg) 3199 .addUse(GR.getSPIRVTypeID(ResType)) 3200 .addUse(I.getOperand(2).getReg()) 3201 .constrainAllUses(TII, TRI, RBI); 3202 } 3203 3204 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3205 const SPIRVType *ResType, 3206 MachineInstr &I) const { 3207 // Change order of instructions if needed: all OpVariable instructions in a 3208 // function must be the first instructions in the first block 3209 auto It = getOpVariableMBBIt(I); 3210 return BuildMI(*It->getParent(), It, It->getDebugLoc(), 3211 TII.get(SPIRV::OpVariable)) 3212 .addDef(ResVReg) 3213 .addUse(GR.getSPIRVTypeID(ResType)) 3214 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3215 .constrainAllUses(TII, TRI, RBI); 3216 } 3217 3218 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3219 // InstructionSelector walks backwards through the instructions. We can use 3220 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3221 // first, so can generate an OpBranchConditional here. If there is no 3222 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3223 const MachineInstr *PrevI = I.getPrevNode(); 3224 MachineBasicBlock &MBB = *I.getParent(); 3225 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3226 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3227 .addUse(PrevI->getOperand(0).getReg()) 3228 .addMBB(PrevI->getOperand(1).getMBB()) 3229 .addMBB(I.getOperand(0).getMBB()) 3230 .constrainAllUses(TII, TRI, RBI); 3231 } 3232 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3233 .addMBB(I.getOperand(0).getMBB()) 3234 .constrainAllUses(TII, TRI, RBI); 3235 } 3236 3237 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3238 // InstructionSelector walks backwards through the instructions. For an 3239 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3240 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3241 // generate the OpBranchConditional in selectBranch above. 3242 // 3243 // If an OpBranchConditional has been generated, we simply return, as the work 3244 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3245 // implicit fallthrough to the next basic block, so we need to create an 3246 // OpBranchConditional with an explicit "false" argument pointing to the next 3247 // basic block that LLVM would fall through to. 3248 const MachineInstr *NextI = I.getNextNode(); 3249 // Check if this has already been successfully selected. 3250 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3251 return true; 3252 // Must be relying on implicit block fallthrough, so generate an 3253 // OpBranchConditional with the "next" basic block as the "false" target. 3254 MachineBasicBlock &MBB = *I.getParent(); 3255 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3256 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3257 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3258 .addUse(I.getOperand(0).getReg()) 3259 .addMBB(I.getOperand(1).getMBB()) 3260 .addMBB(NextMBB) 3261 .constrainAllUses(TII, TRI, RBI); 3262 } 3263 3264 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3265 const SPIRVType *ResType, 3266 MachineInstr &I) const { 3267 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3268 .addDef(ResVReg) 3269 .addUse(GR.getSPIRVTypeID(ResType)); 3270 const unsigned NumOps = I.getNumOperands(); 3271 for (unsigned i = 1; i < NumOps; i += 2) { 3272 MIB.addUse(I.getOperand(i + 0).getReg()); 3273 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3274 } 3275 return MIB.constrainAllUses(TII, TRI, RBI); 3276 } 3277 3278 bool SPIRVInstructionSelector::selectGlobalValue( 3279 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3280 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3281 MachineIRBuilder MIRBuilder(I); 3282 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3283 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3284 SPIRVType *PointerBaseType; 3285 if (GVType->isArrayTy()) { 3286 SPIRVType *ArrayElementType = 3287 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3288 SPIRV::AccessQualifier::ReadWrite, false); 3289 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3290 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3291 } else { 3292 PointerBaseType = GR.getOrCreateSPIRVType( 3293 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3294 } 3295 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3296 PointerBaseType, I, TII, 3297 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3298 3299 std::string GlobalIdent; 3300 if (!GV->hasName()) { 3301 unsigned &ID = UnnamedGlobalIDs[GV]; 3302 if (ID == 0) 3303 ID = UnnamedGlobalIDs.size(); 3304 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3305 } else { 3306 GlobalIdent = GV->getGlobalIdentifier(); 3307 } 3308 3309 // Behaviour of functions as operands depends on availability of the 3310 // corresponding extension (SPV_INTEL_function_pointers): 3311 // - If there is an extension to operate with functions as operands: 3312 // We create a proper constant operand and evaluate a correct type for a 3313 // function pointer. 3314 // - Without the required extension: 3315 // We have functions as operands in tests with blocks of instruction e.g. in 3316 // transcoding/global_block.ll. These operands are not used and should be 3317 // substituted by zero constants. Their type is expected to be always 3318 // OpTypePointer Function %uchar. 3319 if (isa<Function>(GV)) { 3320 const Constant *ConstVal = GV; 3321 MachineBasicBlock &BB = *I.getParent(); 3322 Register NewReg = GR.find(ConstVal, GR.CurMF); 3323 if (!NewReg.isValid()) { 3324 Register NewReg = ResVReg; 3325 GR.add(ConstVal, GR.CurMF, NewReg); 3326 const Function *GVFun = 3327 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3328 ? dyn_cast<Function>(GV) 3329 : nullptr; 3330 if (GVFun) { 3331 // References to a function via function pointers generate virtual 3332 // registers without a definition. We will resolve it later, during 3333 // module analysis stage. 3334 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3335 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3336 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3337 MachineInstrBuilder MB = 3338 BuildMI(BB, I, I.getDebugLoc(), 3339 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3340 .addDef(NewReg) 3341 .addUse(GR.getSPIRVTypeID(ResType)) 3342 .addUse(FuncVReg); 3343 // mapping the function pointer to the used Function 3344 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3345 return MB.constrainAllUses(TII, TRI, RBI); 3346 } 3347 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3348 .addDef(NewReg) 3349 .addUse(GR.getSPIRVTypeID(ResType)) 3350 .constrainAllUses(TII, TRI, RBI); 3351 } 3352 assert(NewReg != ResVReg); 3353 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 3354 .addDef(ResVReg) 3355 .addUse(NewReg) 3356 .constrainAllUses(TII, TRI, RBI); 3357 } 3358 auto GlobalVar = cast<GlobalVariable>(GV); 3359 assert(GlobalVar->getName() != "llvm.global.annotations"); 3360 3361 bool HasInit = GlobalVar->hasInitializer() && 3362 !isa<UndefValue>(GlobalVar->getInitializer()); 3363 // Skip empty declaration for GVs with initilaizers till we get the decl with 3364 // passed initializer. 3365 if (HasInit && !Init) 3366 return true; 3367 3368 unsigned AddrSpace = GV->getAddressSpace(); 3369 SPIRV::StorageClass::StorageClass Storage = 3370 addressSpaceToStorageClass(AddrSpace, STI); 3371 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 3372 Storage != SPIRV::StorageClass::Function; 3373 SPIRV::LinkageType::LinkageType LnkType = 3374 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3375 ? SPIRV::LinkageType::Import 3376 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3377 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3378 ? SPIRV::LinkageType::LinkOnceODR 3379 : SPIRV::LinkageType::Export); 3380 3381 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 3382 Storage, Init, GlobalVar->isConstant(), 3383 HasLnkTy, LnkType, MIRBuilder, true); 3384 return Reg.isValid(); 3385 } 3386 3387 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3388 const SPIRVType *ResType, 3389 MachineInstr &I) const { 3390 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3391 return selectExtInst(ResVReg, ResType, I, CL::log10); 3392 } 3393 3394 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3395 // is implemented as: 3396 // log10(x) = log2(x) * (1 / log2(10)) 3397 // = log2(x) * 0.30103 3398 3399 MachineIRBuilder MIRBuilder(I); 3400 MachineBasicBlock &BB = *I.getParent(); 3401 3402 // Build log2(x). 3403 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3404 bool Result = 3405 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3406 .addDef(VarReg) 3407 .addUse(GR.getSPIRVTypeID(ResType)) 3408 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3409 .addImm(GL::Log2) 3410 .add(I.getOperand(1)) 3411 .constrainAllUses(TII, TRI, RBI); 3412 3413 // Build 0.30103. 3414 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3415 ResType->getOpcode() == SPIRV::OpTypeFloat); 3416 // TODO: Add matrix implementation once supported by the HLSL frontend. 3417 const SPIRVType *SpirvScalarType = 3418 ResType->getOpcode() == SPIRV::OpTypeVector 3419 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3420 : ResType; 3421 Register ScaleReg = 3422 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3423 3424 // Multiply log2(x) by 0.30103 to get log10(x) result. 3425 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3426 ? SPIRV::OpVectorTimesScalar 3427 : SPIRV::OpFMulS; 3428 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3429 .addDef(ResVReg) 3430 .addUse(GR.getSPIRVTypeID(ResType)) 3431 .addUse(VarReg) 3432 .addUse(ScaleReg) 3433 .constrainAllUses(TII, TRI, RBI); 3434 } 3435 3436 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3437 const SPIRVType *ResType, 3438 MachineInstr &I) const { 3439 // DX intrinsic: @llvm.dx.thread.id(i32) 3440 // ID Name Description 3441 // 93 ThreadId reads the thread ID 3442 3443 MachineIRBuilder MIRBuilder(I); 3444 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3445 const SPIRVType *Vec3Ty = 3446 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3447 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3448 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3449 3450 // Create new register for GlobalInvocationID builtin variable. 3451 Register NewRegister = 3452 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3453 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3454 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3455 3456 // Build GlobalInvocationID global variable with the necessary decorations. 3457 Register Variable = GR.buildGlobalVariable( 3458 NewRegister, PtrType, 3459 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3460 SPIRV::StorageClass::Input, nullptr, true, true, 3461 SPIRV::LinkageType::Import, MIRBuilder, false); 3462 3463 // Create new register for loading value. 3464 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3465 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3466 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3467 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3468 3469 // Load v3uint value from the global variable. 3470 bool Result = 3471 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3472 .addDef(LoadedRegister) 3473 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3474 .addUse(Variable); 3475 3476 // Get Thread ID index. Expecting operand is a constant immediate value, 3477 // wrapped in a type assignment. 3478 assert(I.getOperand(2).isReg()); 3479 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3480 3481 // Extract the thread ID from the loaded vector value. 3482 MachineBasicBlock &BB = *I.getParent(); 3483 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3484 .addDef(ResVReg) 3485 .addUse(GR.getSPIRVTypeID(ResType)) 3486 .addUse(LoadedRegister) 3487 .addImm(ThreadId); 3488 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3489 } 3490 3491 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3492 MachineInstr &I) const { 3493 MachineIRBuilder MIRBuilder(I); 3494 if (Type->getOpcode() != SPIRV::OpTypeVector) 3495 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3496 3497 uint64_t VectorSize = Type->getOperand(2).getImm(); 3498 if (VectorSize == 4) 3499 return Type; 3500 3501 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3502 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3503 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3504 } 3505 3506 namespace llvm { 3507 InstructionSelector * 3508 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3509 const SPIRVSubtarget &Subtarget, 3510 const RegisterBankInfo &RBI) { 3511 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3512 } 3513 } // namespace llvm 3514