1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 37 #define DEBUG_TYPE "spirv-isel" 38 39 using namespace llvm; 40 namespace CL = SPIRV::OpenCLExtInst; 41 namespace GL = SPIRV::GLSLExtInst; 42 43 using ExtInstList = 44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 45 46 namespace { 47 48 #define GET_GLOBALISEL_PREDICATE_BITSET 49 #include "SPIRVGenGlobalISel.inc" 50 #undef GET_GLOBALISEL_PREDICATE_BITSET 51 52 class SPIRVInstructionSelector : public InstructionSelector { 53 const SPIRVSubtarget &STI; 54 const SPIRVInstrInfo &TII; 55 const SPIRVRegisterInfo &TRI; 56 const RegisterBankInfo &RBI; 57 SPIRVGlobalRegistry &GR; 58 MachineRegisterInfo *MRI; 59 MachineFunction *HasVRegsReset = nullptr; 60 61 /// We need to keep track of the number we give to anonymous global values to 62 /// generate the same name every time when this is needed. 63 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 64 65 public: 66 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 67 const SPIRVSubtarget &ST, 68 const RegisterBankInfo &RBI); 69 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 70 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 71 BlockFrequencyInfo *BFI) override; 72 // Common selection code. Instruction-specific selection occurs in spvSelect. 73 bool select(MachineInstr &I) override; 74 static const char *getName() { return DEBUG_TYPE; } 75 76 #define GET_GLOBALISEL_PREDICATES_DECL 77 #include "SPIRVGenGlobalISel.inc" 78 #undef GET_GLOBALISEL_PREDICATES_DECL 79 80 #define GET_GLOBALISEL_TEMPORARIES_DECL 81 #include "SPIRVGenGlobalISel.inc" 82 #undef GET_GLOBALISEL_TEMPORARIES_DECL 83 84 private: 85 void resetVRegsType(MachineFunction &MF); 86 87 // tblgen-erated 'select' implementation, used as the initial selector for 88 // the patterns that don't require complex C++. 89 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 90 91 // All instruction-specific selection that didn't happen in "select()". 92 // Is basically a large Switch/Case delegating to all other select method. 93 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 94 MachineInstr &I) const; 95 96 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 97 MachineInstr &I, bool IsSigned) const; 98 99 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 100 MachineInstr &I, bool IsSigned) const; 101 102 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 103 MachineInstr &I, Register SrcReg, 104 bool IsSigned) const; 105 106 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I, bool IsSigned) const; 108 109 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 110 const MachineInstr *Init = nullptr) const; 111 112 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I, std::vector<Register> SrcRegs, 114 unsigned Opcode) const; 115 116 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 117 unsigned Opcode) const; 118 119 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I) const; 121 122 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 123 MachineInstr &I) const; 124 bool selectStore(MachineInstr &I) const; 125 126 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 bool selectStackRestore(MachineInstr &I) const; 129 130 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 131 132 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I, unsigned NewOpcode, 134 unsigned NegateOpcode = 0) const; 135 136 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 137 MachineInstr &I) const; 138 139 bool selectFence(MachineInstr &I) const; 140 141 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 142 MachineInstr &I) const; 143 144 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 145 MachineInstr &I, unsigned OpType) const; 146 147 bool selectAll(Register ResVReg, const SPIRVType *ResType, 148 MachineInstr &I) const; 149 150 bool selectAny(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 153 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 154 MachineInstr &I) const; 155 156 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 157 MachineInstr &I) const; 158 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 162 unsigned comparisonOpcode, MachineInstr &I) const; 163 bool selectCross(Register ResVReg, const SPIRVType *ResType, 164 MachineInstr &I) const; 165 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 168 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectSign(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I, unsigned Opcode) const; 181 182 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I, bool Signed) const; 184 185 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 188 template <bool Signed> 189 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 190 MachineInstr &I) const; 191 template <bool Signed> 192 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 195 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 196 int OpIdx) const; 197 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 198 int OpIdx) const; 199 200 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 201 MachineInstr &I) const; 202 203 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 204 bool IsSigned) const; 205 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 206 bool IsSigned, unsigned Opcode) const; 207 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 208 bool IsSigned) const; 209 210 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 211 MachineInstr &I) const; 212 213 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 214 const SPIRVType *intTy, const SPIRVType *boolTy) const; 215 216 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I) const; 218 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I) const; 220 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I) const; 224 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 227 MachineInstr &I) const; 228 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 229 MachineInstr &I) const; 230 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 233 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 234 MachineInstr &I) const; 235 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 236 MachineInstr &I) const; 237 238 bool selectBranch(MachineInstr &I) const; 239 bool selectBranchCond(MachineInstr &I) const; 240 241 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 242 MachineInstr &I) const; 243 244 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 245 MachineInstr &I, GL::GLSLExtInst GLInst) const; 246 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 247 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 248 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 249 MachineInstr &I, CL::OpenCLExtInst CLInst, 250 GL::GLSLExtInst GLInst) const; 251 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I, const ExtInstList &ExtInsts) const; 253 254 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I) const; 256 257 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I) const; 259 260 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 261 MachineInstr &I) const; 262 263 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 264 MachineInstr &I, unsigned Opcode) const; 265 266 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 267 MachineInstr &I) const; 268 269 bool selectUnmergeValues(MachineInstr &I) const; 270 271 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 272 MachineInstr &I) const; 273 274 void selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 275 MachineInstr &I) const; 276 277 void selectImageWriteIntrinsic(MachineInstr &I) const; 278 279 // Utilities 280 std::pair<Register, bool> 281 buildI32Constant(uint32_t Val, MachineInstr &I, 282 const SPIRVType *ResType = nullptr) const; 283 284 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 285 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 286 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 287 MachineInstr &I) const; 288 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 289 290 bool wrapIntoSpecConstantOp(MachineInstr &I, 291 SmallVector<Register> &CompositeArgs) const; 292 293 Register getUcharPtrTypeReg(MachineInstr &I, 294 SPIRV::StorageClass::StorageClass SC) const; 295 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 296 Register Src, Register DestType, 297 uint32_t Opcode) const; 298 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 299 SPIRVType *SrcPtrTy) const; 300 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 301 uint32_t Binding, uint32_t ArraySize, 302 Register IndexReg, bool IsNonUniform, 303 MachineIRBuilder MIRBuilder) const; 304 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 305 void extractSubvector(Register &ResVReg, const SPIRVType *ResType, 306 Register &ReadReg, MachineInstr &InsertionPoint) const; 307 }; 308 309 } // end anonymous namespace 310 311 #define GET_GLOBALISEL_IMPL 312 #include "SPIRVGenGlobalISel.inc" 313 #undef GET_GLOBALISEL_IMPL 314 315 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 316 const SPIRVSubtarget &ST, 317 const RegisterBankInfo &RBI) 318 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 319 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 320 #define GET_GLOBALISEL_PREDICATES_INIT 321 #include "SPIRVGenGlobalISel.inc" 322 #undef GET_GLOBALISEL_PREDICATES_INIT 323 #define GET_GLOBALISEL_TEMPORARIES_INIT 324 #include "SPIRVGenGlobalISel.inc" 325 #undef GET_GLOBALISEL_TEMPORARIES_INIT 326 { 327 } 328 329 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 330 CodeGenCoverage *CoverageInfo, 331 ProfileSummaryInfo *PSI, 332 BlockFrequencyInfo *BFI) { 333 MRI = &MF.getRegInfo(); 334 GR.setCurrentFunc(MF); 335 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 336 } 337 338 // Ensure that register classes correspond to pattern matching rules. 339 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 340 if (HasVRegsReset == &MF) 341 return; 342 HasVRegsReset = &MF; 343 344 MachineRegisterInfo &MRI = MF.getRegInfo(); 345 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 346 Register Reg = Register::index2VirtReg(I); 347 LLT RegType = MRI.getType(Reg); 348 if (RegType.isScalar()) 349 MRI.setType(Reg, LLT::scalar(64)); 350 else if (RegType.isPointer()) 351 MRI.setType(Reg, LLT::pointer(0, 64)); 352 else if (RegType.isVector()) 353 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 354 } 355 for (const auto &MBB : MF) { 356 for (const auto &MI : MBB) { 357 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 358 continue; 359 Register DstReg = MI.getOperand(0).getReg(); 360 LLT DstType = MRI.getType(DstReg); 361 Register SrcReg = MI.getOperand(1).getReg(); 362 LLT SrcType = MRI.getType(SrcReg); 363 if (DstType != SrcType) 364 MRI.setType(DstReg, MRI.getType(SrcReg)); 365 366 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 367 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 368 if (DstRC != SrcRC && SrcRC) 369 MRI.setRegClass(DstReg, SrcRC); 370 } 371 } 372 } 373 374 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 375 376 // Defined in SPIRVLegalizerInfo.cpp. 377 extern bool isTypeFoldingSupported(unsigned Opcode); 378 379 bool SPIRVInstructionSelector::select(MachineInstr &I) { 380 resetVRegsType(*I.getParent()->getParent()); 381 382 assert(I.getParent() && "Instruction should be in a basic block!"); 383 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 384 385 Register Opcode = I.getOpcode(); 386 // If it's not a GMIR instruction, we've selected it already. 387 if (!isPreISelGenericOpcode(Opcode)) { 388 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 389 Register DstReg = I.getOperand(0).getReg(); 390 Register SrcReg = I.getOperand(1).getReg(); 391 auto *Def = MRI->getVRegDef(SrcReg); 392 if (isTypeFoldingSupported(Def->getOpcode())) { 393 bool Res = selectImpl(I, *CoverageInfo); 394 LLVM_DEBUG({ 395 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 396 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 397 I.print(dbgs()); 398 } 399 }); 400 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 401 if (Res) 402 return Res; 403 } 404 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 405 MRI->replaceRegWith(SrcReg, DstReg); 406 I.removeFromParent(); 407 return true; 408 } else if (I.getNumDefs() == 1) { 409 // Make all vregs 64 bits (for SPIR-V IDs). 410 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 411 } 412 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 413 } 414 415 if (I.getNumOperands() != I.getNumExplicitOperands()) { 416 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 417 return false; 418 } 419 420 // Common code for getting return reg+type, and removing selected instr 421 // from parent occurs here. Instr-specific selection happens in spvSelect(). 422 bool HasDefs = I.getNumDefs() > 0; 423 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 424 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 425 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 426 if (spvSelect(ResVReg, ResType, I)) { 427 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 428 for (unsigned i = 0; i < I.getNumDefs(); ++i) 429 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 430 I.removeFromParent(); 431 return true; 432 } 433 return false; 434 } 435 436 static bool mayApplyGenericSelection(unsigned Opcode) { 437 switch (Opcode) { 438 case TargetOpcode::G_CONSTANT: 439 return false; 440 case TargetOpcode::G_SADDO: 441 case TargetOpcode::G_SSUBO: 442 return true; 443 } 444 return isTypeFoldingSupported(Opcode); 445 } 446 447 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 448 const SPIRVType *ResType, 449 MachineInstr &I) const { 450 const unsigned Opcode = I.getOpcode(); 451 if (mayApplyGenericSelection(Opcode)) 452 return selectImpl(I, *CoverageInfo); 453 switch (Opcode) { 454 case TargetOpcode::G_CONSTANT: 455 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 456 I); 457 case TargetOpcode::G_GLOBAL_VALUE: 458 return selectGlobalValue(ResVReg, I); 459 case TargetOpcode::G_IMPLICIT_DEF: 460 return selectOpUndef(ResVReg, ResType, I); 461 case TargetOpcode::G_FREEZE: 462 return selectFreeze(ResVReg, ResType, I); 463 464 case TargetOpcode::G_INTRINSIC: 465 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 466 case TargetOpcode::G_INTRINSIC_CONVERGENT: 467 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 468 return selectIntrinsic(ResVReg, ResType, I); 469 case TargetOpcode::G_BITREVERSE: 470 return selectBitreverse(ResVReg, ResType, I); 471 472 case TargetOpcode::G_BUILD_VECTOR: 473 return selectBuildVector(ResVReg, ResType, I); 474 case TargetOpcode::G_SPLAT_VECTOR: 475 return selectSplatVector(ResVReg, ResType, I); 476 477 case TargetOpcode::G_SHUFFLE_VECTOR: { 478 MachineBasicBlock &BB = *I.getParent(); 479 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 480 .addDef(ResVReg) 481 .addUse(GR.getSPIRVTypeID(ResType)) 482 .addUse(I.getOperand(1).getReg()) 483 .addUse(I.getOperand(2).getReg()); 484 for (auto V : I.getOperand(3).getShuffleMask()) 485 MIB.addImm(V); 486 return MIB.constrainAllUses(TII, TRI, RBI); 487 } 488 case TargetOpcode::G_MEMMOVE: 489 case TargetOpcode::G_MEMCPY: 490 case TargetOpcode::G_MEMSET: 491 return selectMemOperation(ResVReg, I); 492 493 case TargetOpcode::G_ICMP: 494 return selectICmp(ResVReg, ResType, I); 495 case TargetOpcode::G_FCMP: 496 return selectFCmp(ResVReg, ResType, I); 497 498 case TargetOpcode::G_FRAME_INDEX: 499 return selectFrameIndex(ResVReg, ResType, I); 500 501 case TargetOpcode::G_LOAD: 502 return selectLoad(ResVReg, ResType, I); 503 case TargetOpcode::G_STORE: 504 return selectStore(I); 505 506 case TargetOpcode::G_BR: 507 return selectBranch(I); 508 case TargetOpcode::G_BRCOND: 509 return selectBranchCond(I); 510 511 case TargetOpcode::G_PHI: 512 return selectPhi(ResVReg, ResType, I); 513 514 case TargetOpcode::G_FPTOSI: 515 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 516 case TargetOpcode::G_FPTOUI: 517 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 518 519 case TargetOpcode::G_SITOFP: 520 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 521 case TargetOpcode::G_UITOFP: 522 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 523 524 case TargetOpcode::G_CTPOP: 525 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 526 case TargetOpcode::G_SMIN: 527 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 528 case TargetOpcode::G_UMIN: 529 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 530 531 case TargetOpcode::G_SMAX: 532 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 533 case TargetOpcode::G_UMAX: 534 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 535 536 case TargetOpcode::G_FMA: 537 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 538 539 case TargetOpcode::G_FPOW: 540 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 541 case TargetOpcode::G_FPOWI: 542 return selectExtInst(ResVReg, ResType, I, CL::pown); 543 544 case TargetOpcode::G_FEXP: 545 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 546 case TargetOpcode::G_FEXP2: 547 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 548 549 case TargetOpcode::G_FLOG: 550 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 551 case TargetOpcode::G_FLOG2: 552 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 553 case TargetOpcode::G_FLOG10: 554 return selectLog10(ResVReg, ResType, I); 555 556 case TargetOpcode::G_FABS: 557 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 558 case TargetOpcode::G_ABS: 559 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 560 561 case TargetOpcode::G_FMINNUM: 562 case TargetOpcode::G_FMINIMUM: 563 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 564 case TargetOpcode::G_FMAXNUM: 565 case TargetOpcode::G_FMAXIMUM: 566 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 567 568 case TargetOpcode::G_FCOPYSIGN: 569 return selectExtInst(ResVReg, ResType, I, CL::copysign); 570 571 case TargetOpcode::G_FCEIL: 572 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 573 case TargetOpcode::G_FFLOOR: 574 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 575 576 case TargetOpcode::G_FCOS: 577 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 578 case TargetOpcode::G_FSIN: 579 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 580 case TargetOpcode::G_FTAN: 581 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 582 case TargetOpcode::G_FACOS: 583 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 584 case TargetOpcode::G_FASIN: 585 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 586 case TargetOpcode::G_FATAN: 587 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 588 case TargetOpcode::G_FATAN2: 589 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 590 case TargetOpcode::G_FCOSH: 591 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 592 case TargetOpcode::G_FSINH: 593 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 594 case TargetOpcode::G_FTANH: 595 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 596 597 case TargetOpcode::G_FSQRT: 598 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 599 600 case TargetOpcode::G_CTTZ: 601 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 602 return selectExtInst(ResVReg, ResType, I, CL::ctz); 603 case TargetOpcode::G_CTLZ: 604 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 605 return selectExtInst(ResVReg, ResType, I, CL::clz); 606 607 case TargetOpcode::G_INTRINSIC_ROUND: 608 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 609 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 610 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 611 case TargetOpcode::G_INTRINSIC_TRUNC: 612 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 613 case TargetOpcode::G_FRINT: 614 case TargetOpcode::G_FNEARBYINT: 615 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 616 617 case TargetOpcode::G_SMULH: 618 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 619 case TargetOpcode::G_UMULH: 620 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 621 622 case TargetOpcode::G_SADDSAT: 623 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 624 case TargetOpcode::G_UADDSAT: 625 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 626 case TargetOpcode::G_SSUBSAT: 627 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 628 case TargetOpcode::G_USUBSAT: 629 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 630 631 case TargetOpcode::G_UADDO: 632 return selectOverflowArith(ResVReg, ResType, I, 633 ResType->getOpcode() == SPIRV::OpTypeVector 634 ? SPIRV::OpIAddCarryV 635 : SPIRV::OpIAddCarryS); 636 case TargetOpcode::G_USUBO: 637 return selectOverflowArith(ResVReg, ResType, I, 638 ResType->getOpcode() == SPIRV::OpTypeVector 639 ? SPIRV::OpISubBorrowV 640 : SPIRV::OpISubBorrowS); 641 case TargetOpcode::G_UMULO: 642 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 643 case TargetOpcode::G_SMULO: 644 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 645 646 case TargetOpcode::G_SEXT: 647 return selectExt(ResVReg, ResType, I, true); 648 case TargetOpcode::G_ANYEXT: 649 case TargetOpcode::G_ZEXT: 650 return selectExt(ResVReg, ResType, I, false); 651 case TargetOpcode::G_TRUNC: 652 return selectTrunc(ResVReg, ResType, I); 653 case TargetOpcode::G_FPTRUNC: 654 case TargetOpcode::G_FPEXT: 655 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 656 657 case TargetOpcode::G_PTRTOINT: 658 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 659 case TargetOpcode::G_INTTOPTR: 660 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 661 case TargetOpcode::G_BITCAST: 662 return selectBitcast(ResVReg, ResType, I); 663 case TargetOpcode::G_ADDRSPACE_CAST: 664 return selectAddrSpaceCast(ResVReg, ResType, I); 665 case TargetOpcode::G_PTR_ADD: { 666 // Currently, we get G_PTR_ADD only applied to global variables. 667 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 668 Register GV = I.getOperand(1).getReg(); 669 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 670 (void)II; 671 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 672 (*II).getOpcode() == TargetOpcode::COPY || 673 (*II).getOpcode() == SPIRV::OpVariable) && 674 isImm(I.getOperand(2), MRI)); 675 // It may be the initialization of a global variable. 676 bool IsGVInit = false; 677 for (MachineRegisterInfo::use_instr_iterator 678 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 679 UseEnd = MRI->use_instr_end(); 680 UseIt != UseEnd; UseIt = std::next(UseIt)) { 681 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 682 (*UseIt).getOpcode() == SPIRV::OpVariable) { 683 IsGVInit = true; 684 break; 685 } 686 } 687 MachineBasicBlock &BB = *I.getParent(); 688 if (!IsGVInit) { 689 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 690 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 691 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 692 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 693 // Build a new virtual register that is associated with the required 694 // data type. 695 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 696 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 697 // Having a correctly typed base we are ready to build the actually 698 // required GEP. It may not be a constant though, because all Operands 699 // of OpSpecConstantOp is to originate from other const instructions, 700 // and only the AccessChain named opcodes accept a global OpVariable 701 // instruction. We can't use an AccessChain opcode because of the type 702 // mismatch between result and base types. 703 if (!GR.isBitcastCompatible(ResType, GVType)) 704 report_fatal_error( 705 "incompatible result and operand types in a bitcast"); 706 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 707 MachineInstrBuilder MIB = 708 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 709 .addDef(NewVReg) 710 .addUse(ResTypeReg) 711 .addUse(GV); 712 return MIB.constrainAllUses(TII, TRI, RBI) && 713 BuildMI(BB, I, I.getDebugLoc(), 714 TII.get(STI.isVulkanEnv() 715 ? SPIRV::OpInBoundsAccessChain 716 : SPIRV::OpInBoundsPtrAccessChain)) 717 .addDef(ResVReg) 718 .addUse(ResTypeReg) 719 .addUse(NewVReg) 720 .addUse(I.getOperand(2).getReg()) 721 .constrainAllUses(TII, TRI, RBI); 722 } else { 723 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 724 .addDef(ResVReg) 725 .addUse(GR.getSPIRVTypeID(ResType)) 726 .addImm( 727 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 728 .addUse(GV) 729 .addUse(I.getOperand(2).getReg()) 730 .constrainAllUses(TII, TRI, RBI); 731 } 732 } 733 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 734 // initialize a global variable with a constant expression (e.g., the test 735 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 736 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 737 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 738 .addDef(ResVReg) 739 .addUse(GR.getSPIRVTypeID(ResType)) 740 .addImm(static_cast<uint32_t>( 741 SPIRV::Opcode::InBoundsPtrAccessChain)) 742 .addUse(GV) 743 .addUse(Idx) 744 .addUse(I.getOperand(2).getReg()); 745 return MIB.constrainAllUses(TII, TRI, RBI); 746 } 747 748 case TargetOpcode::G_ATOMICRMW_OR: 749 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 750 case TargetOpcode::G_ATOMICRMW_ADD: 751 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 752 case TargetOpcode::G_ATOMICRMW_AND: 753 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 754 case TargetOpcode::G_ATOMICRMW_MAX: 755 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 756 case TargetOpcode::G_ATOMICRMW_MIN: 757 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 758 case TargetOpcode::G_ATOMICRMW_SUB: 759 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 760 case TargetOpcode::G_ATOMICRMW_XOR: 761 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 762 case TargetOpcode::G_ATOMICRMW_UMAX: 763 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 764 case TargetOpcode::G_ATOMICRMW_UMIN: 765 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 766 case TargetOpcode::G_ATOMICRMW_XCHG: 767 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 768 case TargetOpcode::G_ATOMIC_CMPXCHG: 769 return selectAtomicCmpXchg(ResVReg, ResType, I); 770 771 case TargetOpcode::G_ATOMICRMW_FADD: 772 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 773 case TargetOpcode::G_ATOMICRMW_FSUB: 774 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 775 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 776 SPIRV::OpFNegate); 777 case TargetOpcode::G_ATOMICRMW_FMIN: 778 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 779 case TargetOpcode::G_ATOMICRMW_FMAX: 780 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 781 782 case TargetOpcode::G_FENCE: 783 return selectFence(I); 784 785 case TargetOpcode::G_STACKSAVE: 786 return selectStackSave(ResVReg, ResType, I); 787 case TargetOpcode::G_STACKRESTORE: 788 return selectStackRestore(I); 789 790 case TargetOpcode::G_UNMERGE_VALUES: 791 return selectUnmergeValues(I); 792 793 // Discard gen opcodes for intrinsics which we do not expect to actually 794 // represent code after lowering or intrinsics which are not implemented but 795 // should not crash when found in a customer's LLVM IR input. 796 case TargetOpcode::G_TRAP: 797 case TargetOpcode::G_DEBUGTRAP: 798 case TargetOpcode::G_UBSANTRAP: 799 case TargetOpcode::DBG_LABEL: 800 return true; 801 802 default: 803 return false; 804 } 805 } 806 807 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 808 const SPIRVType *ResType, 809 MachineInstr &I, 810 GL::GLSLExtInst GLInst) const { 811 return selectExtInst(ResVReg, ResType, I, 812 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 813 } 814 815 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 816 const SPIRVType *ResType, 817 MachineInstr &I, 818 CL::OpenCLExtInst CLInst) const { 819 return selectExtInst(ResVReg, ResType, I, 820 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 821 } 822 823 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 824 const SPIRVType *ResType, 825 MachineInstr &I, 826 CL::OpenCLExtInst CLInst, 827 GL::GLSLExtInst GLInst) const { 828 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 829 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 830 return selectExtInst(ResVReg, ResType, I, ExtInsts); 831 } 832 833 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 834 const SPIRVType *ResType, 835 MachineInstr &I, 836 const ExtInstList &Insts) const { 837 838 for (const auto &Ex : Insts) { 839 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 840 uint32_t Opcode = Ex.second; 841 if (STI.canUseExtInstSet(Set)) { 842 MachineBasicBlock &BB = *I.getParent(); 843 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 844 .addDef(ResVReg) 845 .addUse(GR.getSPIRVTypeID(ResType)) 846 .addImm(static_cast<uint32_t>(Set)) 847 .addImm(Opcode); 848 const unsigned NumOps = I.getNumOperands(); 849 unsigned Index = 1; 850 if (Index < NumOps && 851 I.getOperand(Index).getType() == 852 MachineOperand::MachineOperandType::MO_IntrinsicID) 853 Index = 2; 854 for (; Index < NumOps; ++Index) 855 MIB.add(I.getOperand(Index)); 856 return MIB.constrainAllUses(TII, TRI, RBI); 857 } 858 } 859 return false; 860 } 861 862 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 863 const SPIRVType *ResType, 864 MachineInstr &I, 865 std::vector<Register> Srcs, 866 unsigned Opcode) const { 867 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 868 .addDef(ResVReg) 869 .addUse(GR.getSPIRVTypeID(ResType)); 870 for (Register SReg : Srcs) { 871 MIB.addUse(SReg); 872 } 873 return MIB.constrainAllUses(TII, TRI, RBI); 874 } 875 876 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 877 const SPIRVType *ResType, 878 MachineInstr &I, 879 unsigned Opcode) const { 880 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 881 Register SrcReg = I.getOperand(1).getReg(); 882 bool IsGV = false; 883 for (MachineRegisterInfo::def_instr_iterator DefIt = 884 MRI->def_instr_begin(SrcReg); 885 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 886 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 887 IsGV = true; 888 break; 889 } 890 } 891 if (IsGV) { 892 uint32_t SpecOpcode = 0; 893 switch (Opcode) { 894 case SPIRV::OpConvertPtrToU: 895 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 896 break; 897 case SPIRV::OpConvertUToPtr: 898 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 899 break; 900 } 901 if (SpecOpcode) 902 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 903 TII.get(SPIRV::OpSpecConstantOp)) 904 .addDef(ResVReg) 905 .addUse(GR.getSPIRVTypeID(ResType)) 906 .addImm(SpecOpcode) 907 .addUse(SrcReg) 908 .constrainAllUses(TII, TRI, RBI); 909 } 910 } 911 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 912 Opcode); 913 } 914 915 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 916 const SPIRVType *ResType, 917 MachineInstr &I) const { 918 Register OpReg = I.getOperand(1).getReg(); 919 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 920 if (!GR.isBitcastCompatible(ResType, OpType)) 921 report_fatal_error("incompatible result and operand types in a bitcast"); 922 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 923 } 924 925 static void addMemoryOperands(MachineMemOperand *MemOp, 926 MachineInstrBuilder &MIB) { 927 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 928 if (MemOp->isVolatile()) 929 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 930 if (MemOp->isNonTemporal()) 931 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 932 if (MemOp->getAlign().value()) 933 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 934 935 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 936 MIB.addImm(SpvMemOp); 937 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 938 MIB.addImm(MemOp->getAlign().value()); 939 } 940 } 941 942 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 943 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 944 if (Flags & MachineMemOperand::Flags::MOVolatile) 945 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 946 if (Flags & MachineMemOperand::Flags::MONonTemporal) 947 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 948 949 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 950 MIB.addImm(SpvMemOp); 951 } 952 953 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 954 const SPIRVType *ResType, 955 MachineInstr &I) const { 956 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 957 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 958 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 959 .addDef(ResVReg) 960 .addUse(GR.getSPIRVTypeID(ResType)) 961 .addUse(Ptr); 962 if (!I.getNumMemOperands()) { 963 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 964 I.getOpcode() == 965 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 966 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 967 } else { 968 addMemoryOperands(*I.memoperands_begin(), MIB); 969 } 970 return MIB.constrainAllUses(TII, TRI, RBI); 971 } 972 973 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 974 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 975 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 976 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 977 MachineBasicBlock &BB = *I.getParent(); 978 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 979 .addUse(Ptr) 980 .addUse(StoreVal); 981 if (!I.getNumMemOperands()) { 982 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 983 I.getOpcode() == 984 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 985 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 986 } else { 987 addMemoryOperands(*I.memoperands_begin(), MIB); 988 } 989 return MIB.constrainAllUses(TII, TRI, RBI); 990 } 991 992 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 993 const SPIRVType *ResType, 994 MachineInstr &I) const { 995 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 996 report_fatal_error( 997 "llvm.stacksave intrinsic: this instruction requires the following " 998 "SPIR-V extension: SPV_INTEL_variable_length_array", 999 false); 1000 MachineBasicBlock &BB = *I.getParent(); 1001 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1002 .addDef(ResVReg) 1003 .addUse(GR.getSPIRVTypeID(ResType)) 1004 .constrainAllUses(TII, TRI, RBI); 1005 } 1006 1007 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1008 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1009 report_fatal_error( 1010 "llvm.stackrestore intrinsic: this instruction requires the following " 1011 "SPIR-V extension: SPV_INTEL_variable_length_array", 1012 false); 1013 if (!I.getOperand(0).isReg()) 1014 return false; 1015 MachineBasicBlock &BB = *I.getParent(); 1016 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1017 .addUse(I.getOperand(0).getReg()) 1018 .constrainAllUses(TII, TRI, RBI); 1019 } 1020 1021 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1022 MachineInstr &I) const { 1023 MachineBasicBlock &BB = *I.getParent(); 1024 Register SrcReg = I.getOperand(1).getReg(); 1025 bool Result = true; 1026 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1027 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1028 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1029 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1030 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1031 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1032 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1033 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1034 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1035 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1036 Function &CurFunction = GR.CurMF->getFunction(); 1037 Type *LLVMArrTy = 1038 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1039 // Module takes ownership of the global var. 1040 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1041 true, GlobalValue::InternalLinkage, 1042 Constant::getNullValue(LLVMArrTy)); 1043 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1044 GR.add(GV, GR.CurMF, VarReg); 1045 1046 Result &= 1047 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1048 .addDef(VarReg) 1049 .addUse(GR.getSPIRVTypeID(VarTy)) 1050 .addImm(SPIRV::StorageClass::UniformConstant) 1051 .addUse(Const) 1052 .constrainAllUses(TII, TRI, RBI); 1053 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1054 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1055 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1056 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1057 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1058 } 1059 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1060 .addUse(I.getOperand(0).getReg()) 1061 .addUse(SrcReg) 1062 .addUse(I.getOperand(2).getReg()); 1063 if (I.getNumMemOperands()) 1064 addMemoryOperands(*I.memoperands_begin(), MIB); 1065 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1066 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1067 Result &= 1068 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1069 .addUse(MIB->getOperand(0).getReg()) 1070 .constrainAllUses(TII, TRI, RBI); 1071 return Result; 1072 } 1073 1074 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1075 const SPIRVType *ResType, 1076 MachineInstr &I, 1077 unsigned NewOpcode, 1078 unsigned NegateOpcode) const { 1079 bool Result = true; 1080 assert(I.hasOneMemOperand()); 1081 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1082 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1083 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1084 auto ScopeConstant = buildI32Constant(Scope, I); 1085 Register ScopeReg = ScopeConstant.first; 1086 Result &= ScopeConstant.second; 1087 1088 Register Ptr = I.getOperand(1).getReg(); 1089 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1090 // auto ScSem = 1091 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1092 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1093 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1094 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1095 Register MemSemReg = MemSemConstant.first; 1096 Result &= MemSemConstant.second; 1097 1098 Register ValueReg = I.getOperand(2).getReg(); 1099 if (NegateOpcode != 0) { 1100 // Translation with negative value operand is requested 1101 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1102 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1103 ValueReg = TmpReg; 1104 } 1105 1106 return Result && 1107 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1108 .addDef(ResVReg) 1109 .addUse(GR.getSPIRVTypeID(ResType)) 1110 .addUse(Ptr) 1111 .addUse(ScopeReg) 1112 .addUse(MemSemReg) 1113 .addUse(ValueReg) 1114 .constrainAllUses(TII, TRI, RBI); 1115 } 1116 1117 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1118 unsigned ArgI = I.getNumOperands() - 1; 1119 Register SrcReg = 1120 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1121 SPIRVType *DefType = 1122 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1123 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1124 report_fatal_error( 1125 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1126 1127 SPIRVType *ScalarType = 1128 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1129 MachineBasicBlock &BB = *I.getParent(); 1130 bool Res = false; 1131 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1132 Register ResVReg = I.getOperand(i).getReg(); 1133 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1134 if (!ResType) { 1135 // There was no "assign type" actions, let's fix this now 1136 ResType = ScalarType; 1137 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1138 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1139 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1140 } 1141 auto MIB = 1142 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1143 .addDef(ResVReg) 1144 .addUse(GR.getSPIRVTypeID(ResType)) 1145 .addUse(SrcReg) 1146 .addImm(static_cast<int64_t>(i)); 1147 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1148 } 1149 return Res; 1150 } 1151 1152 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1153 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1154 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1155 auto MemSemConstant = buildI32Constant(MemSem, I); 1156 Register MemSemReg = MemSemConstant.first; 1157 bool Result = MemSemConstant.second; 1158 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1159 uint32_t Scope = static_cast<uint32_t>( 1160 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1161 auto ScopeConstant = buildI32Constant(Scope, I); 1162 Register ScopeReg = ScopeConstant.first; 1163 Result &= ScopeConstant.second; 1164 MachineBasicBlock &BB = *I.getParent(); 1165 return Result && 1166 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1167 .addUse(ScopeReg) 1168 .addUse(MemSemReg) 1169 .constrainAllUses(TII, TRI, RBI); 1170 } 1171 1172 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1173 const SPIRVType *ResType, 1174 MachineInstr &I, 1175 unsigned Opcode) const { 1176 Type *ResTy = nullptr; 1177 StringRef ResName; 1178 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1179 report_fatal_error( 1180 "Not enough info to select the arithmetic with overflow instruction"); 1181 if (!ResTy || !ResTy->isStructTy()) 1182 report_fatal_error("Expect struct type result for the arithmetic " 1183 "with overflow instruction"); 1184 // "Result Type must be from OpTypeStruct. The struct must have two members, 1185 // and the two members must be the same type." 1186 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1187 ResTy = StructType::get(ResElemTy, ResElemTy); 1188 // Build SPIR-V types and constant(s) if needed. 1189 MachineIRBuilder MIRBuilder(I); 1190 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1191 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1192 assert(I.getNumDefs() > 1 && "Not enought operands"); 1193 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1194 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1195 if (N > 1) 1196 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1197 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1198 Register ZeroReg = buildZerosVal(ResType, I); 1199 // A new virtual register to store the result struct. 1200 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1201 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1202 // Build the result name if needed. 1203 if (ResName.size() > 0) 1204 buildOpName(StructVReg, ResName, MIRBuilder); 1205 // Build the arithmetic with overflow instruction. 1206 MachineBasicBlock &BB = *I.getParent(); 1207 auto MIB = 1208 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1209 .addDef(StructVReg) 1210 .addUse(GR.getSPIRVTypeID(StructType)); 1211 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1212 MIB.addUse(I.getOperand(i).getReg()); 1213 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1214 // Build instructions to extract fields of the instruction's result. 1215 // A new virtual register to store the higher part of the result struct. 1216 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1217 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1218 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1219 auto MIB = 1220 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1221 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1222 .addUse(GR.getSPIRVTypeID(ResType)) 1223 .addUse(StructVReg) 1224 .addImm(i); 1225 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1226 } 1227 // Build boolean value from the higher part. 1228 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1229 .addDef(I.getOperand(1).getReg()) 1230 .addUse(BoolTypeReg) 1231 .addUse(HigherVReg) 1232 .addUse(ZeroReg) 1233 .constrainAllUses(TII, TRI, RBI); 1234 } 1235 1236 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1237 const SPIRVType *ResType, 1238 MachineInstr &I) const { 1239 bool Result = true; 1240 Register ScopeReg; 1241 Register MemSemEqReg; 1242 Register MemSemNeqReg; 1243 Register Ptr = I.getOperand(2).getReg(); 1244 if (!isa<GIntrinsic>(I)) { 1245 assert(I.hasOneMemOperand()); 1246 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1247 unsigned Scope = static_cast<uint32_t>(getMemScope( 1248 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1249 auto ScopeConstant = buildI32Constant(Scope, I); 1250 ScopeReg = ScopeConstant.first; 1251 Result &= ScopeConstant.second; 1252 1253 unsigned ScSem = static_cast<uint32_t>( 1254 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1255 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1256 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1257 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1258 MemSemEqReg = MemSemEqConstant.first; 1259 Result &= MemSemEqConstant.second; 1260 AtomicOrdering FO = MemOp->getFailureOrdering(); 1261 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1262 if (MemSemEq == MemSemNeq) 1263 MemSemNeqReg = MemSemEqReg; 1264 else { 1265 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1266 MemSemNeqReg = MemSemNeqConstant.first; 1267 Result &= MemSemNeqConstant.second; 1268 } 1269 } else { 1270 ScopeReg = I.getOperand(5).getReg(); 1271 MemSemEqReg = I.getOperand(6).getReg(); 1272 MemSemNeqReg = I.getOperand(7).getReg(); 1273 } 1274 1275 Register Cmp = I.getOperand(3).getReg(); 1276 Register Val = I.getOperand(4).getReg(); 1277 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1278 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1279 const DebugLoc &DL = I.getDebugLoc(); 1280 Result &= 1281 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1282 .addDef(ACmpRes) 1283 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1284 .addUse(Ptr) 1285 .addUse(ScopeReg) 1286 .addUse(MemSemEqReg) 1287 .addUse(MemSemNeqReg) 1288 .addUse(Val) 1289 .addUse(Cmp) 1290 .constrainAllUses(TII, TRI, RBI); 1291 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1292 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1293 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1294 .addDef(CmpSuccReg) 1295 .addUse(GR.getSPIRVTypeID(BoolTy)) 1296 .addUse(ACmpRes) 1297 .addUse(Cmp) 1298 .constrainAllUses(TII, TRI, RBI); 1299 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1300 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1301 .addDef(TmpReg) 1302 .addUse(GR.getSPIRVTypeID(ResType)) 1303 .addUse(ACmpRes) 1304 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1305 .addImm(0) 1306 .constrainAllUses(TII, TRI, RBI); 1307 return Result && 1308 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1309 .addDef(ResVReg) 1310 .addUse(GR.getSPIRVTypeID(ResType)) 1311 .addUse(CmpSuccReg) 1312 .addUse(TmpReg) 1313 .addImm(1) 1314 .constrainAllUses(TII, TRI, RBI); 1315 } 1316 1317 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1318 switch (SC) { 1319 case SPIRV::StorageClass::Workgroup: 1320 case SPIRV::StorageClass::CrossWorkgroup: 1321 case SPIRV::StorageClass::Function: 1322 return true; 1323 default: 1324 return false; 1325 } 1326 } 1327 1328 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1329 switch (SC) { 1330 case SPIRV::StorageClass::DeviceOnlyINTEL: 1331 case SPIRV::StorageClass::HostOnlyINTEL: 1332 return true; 1333 default: 1334 return false; 1335 } 1336 } 1337 1338 // Returns true ResVReg is referred only from global vars and OpName's. 1339 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1340 bool IsGRef = false; 1341 bool IsAllowedRefs = 1342 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1343 [&IsGRef](auto const &It) { 1344 unsigned Opcode = It.getOpcode(); 1345 if (Opcode == SPIRV::OpConstantComposite || 1346 Opcode == SPIRV::OpVariable || 1347 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1348 return IsGRef = true; 1349 return Opcode == SPIRV::OpName; 1350 }); 1351 return IsAllowedRefs && IsGRef; 1352 } 1353 1354 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1355 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1356 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1357 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1358 } 1359 1360 MachineInstrBuilder 1361 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1362 Register Src, Register DestType, 1363 uint32_t Opcode) const { 1364 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1365 TII.get(SPIRV::OpSpecConstantOp)) 1366 .addDef(Dest) 1367 .addUse(DestType) 1368 .addImm(Opcode) 1369 .addUse(Src); 1370 } 1371 1372 MachineInstrBuilder 1373 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1374 SPIRVType *SrcPtrTy) const { 1375 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1376 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1377 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1378 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1379 SPIRV::StorageClass::Generic), 1380 GR.getPointerSize())); 1381 MachineFunction *MF = I.getParent()->getParent(); 1382 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1383 MachineInstrBuilder MIB = buildSpecConstantOp( 1384 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1385 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1386 GR.add(MIB.getInstr(), MF, Tmp); 1387 return MIB; 1388 } 1389 1390 // In SPIR-V address space casting can only happen to and from the Generic 1391 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1392 // pointers to and from Generic pointers. As such, we can convert e.g. from 1393 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1394 // other combinations can only be done by a bitcast, and are probably not safe. 1395 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1396 const SPIRVType *ResType, 1397 MachineInstr &I) const { 1398 MachineBasicBlock &BB = *I.getParent(); 1399 const DebugLoc &DL = I.getDebugLoc(); 1400 1401 Register SrcPtr = I.getOperand(1).getReg(); 1402 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1403 1404 // don't generate a cast for a null that may be represented by OpTypeInt 1405 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1406 ResType->getOpcode() != SPIRV::OpTypePointer) 1407 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1408 .addDef(ResVReg) 1409 .addUse(SrcPtr) 1410 .constrainAllUses(TII, TRI, RBI); 1411 1412 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1413 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1414 1415 if (isASCastInGVar(MRI, ResVReg)) { 1416 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1417 // are expressed by OpSpecConstantOp with an Opcode. 1418 // TODO: maybe insert a check whether the Kernel capability was declared and 1419 // so PtrCastToGeneric/GenericCastToPtr are available. 1420 unsigned SpecOpcode = 1421 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1422 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1423 : (SrcSC == SPIRV::StorageClass::Generic && 1424 isGenericCastablePtr(DstSC) 1425 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1426 : 0); 1427 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1428 // correct value of ResType and use general i8* instead. Maybe this should 1429 // be addressed in the emit-intrinsic step to infer a correct 1430 // OpConstantComposite type. 1431 if (SpecOpcode) { 1432 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1433 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1434 .constrainAllUses(TII, TRI, RBI); 1435 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1436 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1437 return MIB.constrainAllUses(TII, TRI, RBI) && 1438 buildSpecConstantOp( 1439 I, ResVReg, MIB->getOperand(0).getReg(), 1440 getUcharPtrTypeReg(I, DstSC), 1441 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1442 .constrainAllUses(TII, TRI, RBI); 1443 } 1444 } 1445 1446 // don't generate a cast between identical storage classes 1447 if (SrcSC == DstSC) 1448 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1449 .addDef(ResVReg) 1450 .addUse(SrcPtr) 1451 .constrainAllUses(TII, TRI, RBI); 1452 1453 // Casting from an eligible pointer to Generic. 1454 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1455 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1456 // Casting from Generic to an eligible pointer. 1457 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1458 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1459 // Casting between 2 eligible pointers using Generic as an intermediary. 1460 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1461 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1462 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1463 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1464 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1465 .addDef(Tmp) 1466 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1467 .addUse(SrcPtr) 1468 .constrainAllUses(TII, TRI, RBI); 1469 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1470 .addDef(ResVReg) 1471 .addUse(GR.getSPIRVTypeID(ResType)) 1472 .addUse(Tmp) 1473 .constrainAllUses(TII, TRI, RBI); 1474 } 1475 1476 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1477 // be applied 1478 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1479 return selectUnOp(ResVReg, ResType, I, 1480 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1481 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1482 return selectUnOp(ResVReg, ResType, I, 1483 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1484 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1485 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1486 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1487 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1488 1489 // Bitcast for pointers requires that the address spaces must match 1490 return false; 1491 } 1492 1493 static unsigned getFCmpOpcode(unsigned PredNum) { 1494 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1495 switch (Pred) { 1496 case CmpInst::FCMP_OEQ: 1497 return SPIRV::OpFOrdEqual; 1498 case CmpInst::FCMP_OGE: 1499 return SPIRV::OpFOrdGreaterThanEqual; 1500 case CmpInst::FCMP_OGT: 1501 return SPIRV::OpFOrdGreaterThan; 1502 case CmpInst::FCMP_OLE: 1503 return SPIRV::OpFOrdLessThanEqual; 1504 case CmpInst::FCMP_OLT: 1505 return SPIRV::OpFOrdLessThan; 1506 case CmpInst::FCMP_ONE: 1507 return SPIRV::OpFOrdNotEqual; 1508 case CmpInst::FCMP_ORD: 1509 return SPIRV::OpOrdered; 1510 case CmpInst::FCMP_UEQ: 1511 return SPIRV::OpFUnordEqual; 1512 case CmpInst::FCMP_UGE: 1513 return SPIRV::OpFUnordGreaterThanEqual; 1514 case CmpInst::FCMP_UGT: 1515 return SPIRV::OpFUnordGreaterThan; 1516 case CmpInst::FCMP_ULE: 1517 return SPIRV::OpFUnordLessThanEqual; 1518 case CmpInst::FCMP_ULT: 1519 return SPIRV::OpFUnordLessThan; 1520 case CmpInst::FCMP_UNE: 1521 return SPIRV::OpFUnordNotEqual; 1522 case CmpInst::FCMP_UNO: 1523 return SPIRV::OpUnordered; 1524 default: 1525 llvm_unreachable("Unknown predicate type for FCmp"); 1526 } 1527 } 1528 1529 static unsigned getICmpOpcode(unsigned PredNum) { 1530 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1531 switch (Pred) { 1532 case CmpInst::ICMP_EQ: 1533 return SPIRV::OpIEqual; 1534 case CmpInst::ICMP_NE: 1535 return SPIRV::OpINotEqual; 1536 case CmpInst::ICMP_SGE: 1537 return SPIRV::OpSGreaterThanEqual; 1538 case CmpInst::ICMP_SGT: 1539 return SPIRV::OpSGreaterThan; 1540 case CmpInst::ICMP_SLE: 1541 return SPIRV::OpSLessThanEqual; 1542 case CmpInst::ICMP_SLT: 1543 return SPIRV::OpSLessThan; 1544 case CmpInst::ICMP_UGE: 1545 return SPIRV::OpUGreaterThanEqual; 1546 case CmpInst::ICMP_UGT: 1547 return SPIRV::OpUGreaterThan; 1548 case CmpInst::ICMP_ULE: 1549 return SPIRV::OpULessThanEqual; 1550 case CmpInst::ICMP_ULT: 1551 return SPIRV::OpULessThan; 1552 default: 1553 llvm_unreachable("Unknown predicate type for ICmp"); 1554 } 1555 } 1556 1557 static unsigned getPtrCmpOpcode(unsigned Pred) { 1558 switch (static_cast<CmpInst::Predicate>(Pred)) { 1559 case CmpInst::ICMP_EQ: 1560 return SPIRV::OpPtrEqual; 1561 case CmpInst::ICMP_NE: 1562 return SPIRV::OpPtrNotEqual; 1563 default: 1564 llvm_unreachable("Unknown predicate type for pointer comparison"); 1565 } 1566 } 1567 1568 // Return the logical operation, or abort if none exists. 1569 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1570 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1571 switch (Pred) { 1572 case CmpInst::ICMP_EQ: 1573 return SPIRV::OpLogicalEqual; 1574 case CmpInst::ICMP_NE: 1575 return SPIRV::OpLogicalNotEqual; 1576 default: 1577 llvm_unreachable("Unknown predicate type for Bool comparison"); 1578 } 1579 } 1580 1581 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1582 if (!LLVMFloatTy) 1583 return APFloat::getZero(APFloat::IEEEsingle()); 1584 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1585 case Type::HalfTyID: 1586 return APFloat::getZero(APFloat::IEEEhalf()); 1587 default: 1588 case Type::FloatTyID: 1589 return APFloat::getZero(APFloat::IEEEsingle()); 1590 case Type::DoubleTyID: 1591 return APFloat::getZero(APFloat::IEEEdouble()); 1592 } 1593 } 1594 1595 static APFloat getOneFP(const Type *LLVMFloatTy) { 1596 if (!LLVMFloatTy) 1597 return APFloat::getOne(APFloat::IEEEsingle()); 1598 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1599 case Type::HalfTyID: 1600 return APFloat::getOne(APFloat::IEEEhalf()); 1601 default: 1602 case Type::FloatTyID: 1603 return APFloat::getOne(APFloat::IEEEsingle()); 1604 case Type::DoubleTyID: 1605 return APFloat::getOne(APFloat::IEEEdouble()); 1606 } 1607 } 1608 1609 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1610 const SPIRVType *ResType, 1611 MachineInstr &I, 1612 unsigned OpAnyOrAll) const { 1613 assert(I.getNumOperands() == 3); 1614 assert(I.getOperand(2).isReg()); 1615 MachineBasicBlock &BB = *I.getParent(); 1616 Register InputRegister = I.getOperand(2).getReg(); 1617 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1618 1619 if (!InputType) 1620 report_fatal_error("Input Type could not be determined."); 1621 1622 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1623 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1624 if (IsBoolTy && !IsVectorTy) { 1625 assert(ResVReg == I.getOperand(0).getReg()); 1626 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1627 TII.get(TargetOpcode::COPY)) 1628 .addDef(ResVReg) 1629 .addUse(InputRegister) 1630 .constrainAllUses(TII, TRI, RBI); 1631 } 1632 1633 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1634 unsigned SpirvNotEqualId = 1635 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1636 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1637 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1638 Register NotEqualReg = ResVReg; 1639 1640 if (IsVectorTy) { 1641 NotEqualReg = IsBoolTy ? InputRegister 1642 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1643 const unsigned NumElts = InputType->getOperand(2).getImm(); 1644 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1645 } 1646 1647 bool Result = true; 1648 if (!IsBoolTy) { 1649 Register ConstZeroReg = 1650 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1651 1652 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1653 .addDef(NotEqualReg) 1654 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1655 .addUse(InputRegister) 1656 .addUse(ConstZeroReg) 1657 .constrainAllUses(TII, TRI, RBI); 1658 } 1659 1660 if (!IsVectorTy) 1661 return Result; 1662 1663 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1664 .addDef(ResVReg) 1665 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1666 .addUse(NotEqualReg) 1667 .constrainAllUses(TII, TRI, RBI); 1668 } 1669 1670 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1671 const SPIRVType *ResType, 1672 MachineInstr &I) const { 1673 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1674 } 1675 1676 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1677 const SPIRVType *ResType, 1678 MachineInstr &I) const { 1679 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1680 } 1681 1682 // Select the OpDot instruction for the given float dot 1683 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1684 const SPIRVType *ResType, 1685 MachineInstr &I) const { 1686 assert(I.getNumOperands() == 4); 1687 assert(I.getOperand(2).isReg()); 1688 assert(I.getOperand(3).isReg()); 1689 1690 [[maybe_unused]] SPIRVType *VecType = 1691 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1692 1693 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1694 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1695 "dot product requires a vector of at least 2 components"); 1696 1697 [[maybe_unused]] SPIRVType *EltType = 1698 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1699 1700 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1701 1702 MachineBasicBlock &BB = *I.getParent(); 1703 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1704 .addDef(ResVReg) 1705 .addUse(GR.getSPIRVTypeID(ResType)) 1706 .addUse(I.getOperand(2).getReg()) 1707 .addUse(I.getOperand(3).getReg()) 1708 .constrainAllUses(TII, TRI, RBI); 1709 } 1710 1711 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1712 const SPIRVType *ResType, 1713 MachineInstr &I, 1714 bool Signed) const { 1715 assert(I.getNumOperands() == 4); 1716 assert(I.getOperand(2).isReg()); 1717 assert(I.getOperand(3).isReg()); 1718 MachineBasicBlock &BB = *I.getParent(); 1719 1720 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1721 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1722 .addDef(ResVReg) 1723 .addUse(GR.getSPIRVTypeID(ResType)) 1724 .addUse(I.getOperand(2).getReg()) 1725 .addUse(I.getOperand(3).getReg()) 1726 .constrainAllUses(TII, TRI, RBI); 1727 } 1728 1729 // Since pre-1.6 SPIRV has no integer dot implementation, 1730 // expand by piecewise multiplying and adding the results 1731 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1732 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1733 assert(I.getNumOperands() == 4); 1734 assert(I.getOperand(2).isReg()); 1735 assert(I.getOperand(3).isReg()); 1736 MachineBasicBlock &BB = *I.getParent(); 1737 1738 // Multiply the vectors, then sum the results 1739 Register Vec0 = I.getOperand(2).getReg(); 1740 Register Vec1 = I.getOperand(3).getReg(); 1741 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1742 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1743 1744 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1745 .addDef(TmpVec) 1746 .addUse(GR.getSPIRVTypeID(VecType)) 1747 .addUse(Vec0) 1748 .addUse(Vec1) 1749 .constrainAllUses(TII, TRI, RBI); 1750 1751 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1752 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1753 "dot product requires a vector of at least 2 components"); 1754 1755 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1756 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1757 .addDef(Res) 1758 .addUse(GR.getSPIRVTypeID(ResType)) 1759 .addUse(TmpVec) 1760 .addImm(0) 1761 .constrainAllUses(TII, TRI, RBI); 1762 1763 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1764 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1765 1766 Result &= 1767 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1768 .addDef(Elt) 1769 .addUse(GR.getSPIRVTypeID(ResType)) 1770 .addUse(TmpVec) 1771 .addImm(i) 1772 .constrainAllUses(TII, TRI, RBI); 1773 1774 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1775 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1776 : ResVReg; 1777 1778 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1779 .addDef(Sum) 1780 .addUse(GR.getSPIRVTypeID(ResType)) 1781 .addUse(Res) 1782 .addUse(Elt) 1783 .constrainAllUses(TII, TRI, RBI); 1784 Res = Sum; 1785 } 1786 1787 return Result; 1788 } 1789 1790 template <bool Signed> 1791 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1792 const SPIRVType *ResType, 1793 MachineInstr &I) const { 1794 assert(I.getNumOperands() == 5); 1795 assert(I.getOperand(2).isReg()); 1796 assert(I.getOperand(3).isReg()); 1797 assert(I.getOperand(4).isReg()); 1798 MachineBasicBlock &BB = *I.getParent(); 1799 1800 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1801 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1802 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1803 .addDef(Dot) 1804 .addUse(GR.getSPIRVTypeID(ResType)) 1805 .addUse(I.getOperand(2).getReg()) 1806 .addUse(I.getOperand(3).getReg()) 1807 .constrainAllUses(TII, TRI, RBI); 1808 1809 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1810 .addDef(ResVReg) 1811 .addUse(GR.getSPIRVTypeID(ResType)) 1812 .addUse(Dot) 1813 .addUse(I.getOperand(4).getReg()) 1814 .constrainAllUses(TII, TRI, RBI); 1815 } 1816 1817 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1818 // extract the elements of the packed inputs, multiply them and add the result 1819 // to the accumulator. 1820 template <bool Signed> 1821 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1822 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1823 assert(I.getNumOperands() == 5); 1824 assert(I.getOperand(2).isReg()); 1825 assert(I.getOperand(3).isReg()); 1826 assert(I.getOperand(4).isReg()); 1827 MachineBasicBlock &BB = *I.getParent(); 1828 1829 bool Result = true; 1830 1831 // Acc = C 1832 Register Acc = I.getOperand(4).getReg(); 1833 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1834 auto ExtractOp = 1835 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1836 1837 // Extract the i8 element, multiply and add it to the accumulator 1838 for (unsigned i = 0; i < 4; i++) { 1839 // A[i] 1840 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1841 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1842 .addDef(AElt) 1843 .addUse(GR.getSPIRVTypeID(ResType)) 1844 .addUse(I.getOperand(2).getReg()) 1845 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1846 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1847 .constrainAllUses(TII, TRI, RBI); 1848 1849 // B[i] 1850 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1851 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1852 .addDef(BElt) 1853 .addUse(GR.getSPIRVTypeID(ResType)) 1854 .addUse(I.getOperand(3).getReg()) 1855 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1856 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1857 .constrainAllUses(TII, TRI, RBI); 1858 1859 // A[i] * B[i] 1860 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1861 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1862 .addDef(Mul) 1863 .addUse(GR.getSPIRVTypeID(ResType)) 1864 .addUse(AElt) 1865 .addUse(BElt) 1866 .constrainAllUses(TII, TRI, RBI); 1867 1868 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1869 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1870 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1871 .addDef(MaskMul) 1872 .addUse(GR.getSPIRVTypeID(ResType)) 1873 .addUse(Mul) 1874 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1875 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1876 .constrainAllUses(TII, TRI, RBI); 1877 1878 // Acc = Acc + A[i] * B[i] 1879 Register Sum = 1880 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1881 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1882 .addDef(Sum) 1883 .addUse(GR.getSPIRVTypeID(ResType)) 1884 .addUse(Acc) 1885 .addUse(MaskMul) 1886 .constrainAllUses(TII, TRI, RBI); 1887 1888 Acc = Sum; 1889 } 1890 1891 return Result; 1892 } 1893 1894 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1895 /// does not have a saturate builtin. 1896 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1897 const SPIRVType *ResType, 1898 MachineInstr &I) const { 1899 assert(I.getNumOperands() == 3); 1900 assert(I.getOperand(2).isReg()); 1901 MachineBasicBlock &BB = *I.getParent(); 1902 Register VZero = buildZerosValF(ResType, I); 1903 Register VOne = buildOnesValF(ResType, I); 1904 1905 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1906 .addDef(ResVReg) 1907 .addUse(GR.getSPIRVTypeID(ResType)) 1908 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1909 .addImm(GL::FClamp) 1910 .addUse(I.getOperand(2).getReg()) 1911 .addUse(VZero) 1912 .addUse(VOne) 1913 .constrainAllUses(TII, TRI, RBI); 1914 } 1915 1916 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1917 const SPIRVType *ResType, 1918 MachineInstr &I) const { 1919 assert(I.getNumOperands() == 3); 1920 assert(I.getOperand(2).isReg()); 1921 MachineBasicBlock &BB = *I.getParent(); 1922 Register InputRegister = I.getOperand(2).getReg(); 1923 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1924 auto &DL = I.getDebugLoc(); 1925 1926 if (!InputType) 1927 report_fatal_error("Input Type could not be determined."); 1928 1929 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1930 1931 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1932 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1933 1934 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1935 1936 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1937 Register SignReg = NeedsConversion 1938 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1939 : ResVReg; 1940 1941 bool Result = 1942 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1943 .addDef(SignReg) 1944 .addUse(GR.getSPIRVTypeID(InputType)) 1945 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1946 .addImm(SignOpcode) 1947 .addUse(InputRegister) 1948 .constrainAllUses(TII, TRI, RBI); 1949 1950 if (NeedsConversion) { 1951 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1952 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1953 .addDef(ResVReg) 1954 .addUse(GR.getSPIRVTypeID(ResType)) 1955 .addUse(SignReg) 1956 .constrainAllUses(TII, TRI, RBI); 1957 } 1958 1959 return Result; 1960 } 1961 1962 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 1963 const SPIRVType *ResType, 1964 MachineInstr &I, 1965 unsigned Opcode) const { 1966 MachineBasicBlock &BB = *I.getParent(); 1967 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1968 1969 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1970 .addDef(ResVReg) 1971 .addUse(GR.getSPIRVTypeID(ResType)) 1972 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 1973 IntTy, TII)); 1974 1975 for (unsigned J = 2; J < I.getNumOperands(); J++) { 1976 BMI.addUse(I.getOperand(J).getReg()); 1977 } 1978 1979 return BMI.constrainAllUses(TII, TRI, RBI); 1980 } 1981 1982 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 1983 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1984 1985 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1986 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 1987 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 1988 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 1989 SPIRV::OpGroupNonUniformBallot); 1990 1991 MachineBasicBlock &BB = *I.getParent(); 1992 Result &= 1993 BuildMI(BB, I, I.getDebugLoc(), 1994 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 1995 .addDef(ResVReg) 1996 .addUse(GR.getSPIRVTypeID(ResType)) 1997 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1998 .addImm(SPIRV::GroupOperation::Reduce) 1999 .addUse(BallotReg) 2000 .constrainAllUses(TII, TRI, RBI); 2001 2002 return Result; 2003 } 2004 2005 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2006 const SPIRVType *ResType, 2007 MachineInstr &I) const { 2008 MachineBasicBlock &BB = *I.getParent(); 2009 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2010 .addDef(ResVReg) 2011 .addUse(GR.getSPIRVTypeID(ResType)) 2012 .addUse(I.getOperand(1).getReg()) 2013 .constrainAllUses(TII, TRI, RBI); 2014 } 2015 2016 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2017 const SPIRVType *ResType, 2018 MachineInstr &I) const { 2019 // There is no way to implement `freeze` correctly without support on SPIR-V 2020 // standard side, but we may at least address a simple (static) case when 2021 // undef/poison value presence is obvious. The main benefit of even 2022 // incomplete `freeze` support is preventing of translation from crashing due 2023 // to lack of support on legalization and instruction selection steps. 2024 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2025 return false; 2026 Register OpReg = I.getOperand(1).getReg(); 2027 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2028 Register Reg; 2029 switch (Def->getOpcode()) { 2030 case SPIRV::ASSIGN_TYPE: 2031 if (MachineInstr *AssignToDef = 2032 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2033 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2034 Reg = Def->getOperand(2).getReg(); 2035 } 2036 break; 2037 case SPIRV::OpUndef: 2038 Reg = Def->getOperand(1).getReg(); 2039 break; 2040 } 2041 unsigned DestOpCode; 2042 if (Reg.isValid()) { 2043 DestOpCode = SPIRV::OpConstantNull; 2044 } else { 2045 DestOpCode = TargetOpcode::COPY; 2046 Reg = OpReg; 2047 } 2048 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2049 .addDef(I.getOperand(0).getReg()) 2050 .addUse(Reg) 2051 .constrainAllUses(TII, TRI, RBI); 2052 } 2053 return false; 2054 } 2055 2056 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2057 const SPIRVType *ResType) { 2058 Register OpReg = ResType->getOperand(2).getReg(); 2059 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2060 if (!OpDef) 2061 return 0; 2062 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2063 OpDef->getOperand(1).isReg()) { 2064 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2065 OpDef = RefDef; 2066 } 2067 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2068 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2069 : 0; 2070 return N; 2071 } 2072 2073 // Return true if the type represents a constant register 2074 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2075 SmallPtrSet<SPIRVType *, 4> &Visited) { 2076 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2077 OpDef->getOperand(1).isReg()) { 2078 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2079 OpDef = RefDef; 2080 } 2081 2082 if (Visited.contains(OpDef)) 2083 return true; 2084 Visited.insert(OpDef); 2085 2086 unsigned Opcode = OpDef->getOpcode(); 2087 switch (Opcode) { 2088 case TargetOpcode::G_CONSTANT: 2089 case TargetOpcode::G_FCONSTANT: 2090 return true; 2091 case TargetOpcode::G_INTRINSIC: 2092 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2093 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2094 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2095 Intrinsic::spv_const_composite; 2096 case TargetOpcode::G_BUILD_VECTOR: 2097 case TargetOpcode::G_SPLAT_VECTOR: { 2098 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2099 i++) { 2100 SPIRVType *OpNestedDef = 2101 OpDef->getOperand(i).isReg() 2102 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2103 : nullptr; 2104 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2105 return false; 2106 } 2107 return true; 2108 } 2109 } 2110 return false; 2111 } 2112 2113 // Return true if the virtual register represents a constant 2114 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2115 SmallPtrSet<SPIRVType *, 4> Visited; 2116 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2117 return isConstReg(MRI, OpDef, Visited); 2118 return false; 2119 } 2120 2121 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2122 const SPIRVType *ResType, 2123 MachineInstr &I) const { 2124 unsigned N = 0; 2125 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2126 N = GR.getScalarOrVectorComponentCount(ResType); 2127 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2128 N = getArrayComponentCount(MRI, ResType); 2129 else 2130 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2131 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2132 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2133 2134 // check if we may construct a constant vector 2135 bool IsConst = true; 2136 for (unsigned i = I.getNumExplicitDefs(); 2137 i < I.getNumExplicitOperands() && IsConst; ++i) 2138 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2139 IsConst = false; 2140 2141 if (!IsConst && N < 2) 2142 report_fatal_error( 2143 "There must be at least two constituent operands in a vector"); 2144 2145 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2146 TII.get(IsConst ? SPIRV::OpConstantComposite 2147 : SPIRV::OpCompositeConstruct)) 2148 .addDef(ResVReg) 2149 .addUse(GR.getSPIRVTypeID(ResType)); 2150 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2151 MIB.addUse(I.getOperand(i).getReg()); 2152 return MIB.constrainAllUses(TII, TRI, RBI); 2153 } 2154 2155 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2156 const SPIRVType *ResType, 2157 MachineInstr &I) const { 2158 unsigned N = 0; 2159 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2160 N = GR.getScalarOrVectorComponentCount(ResType); 2161 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2162 N = getArrayComponentCount(MRI, ResType); 2163 else 2164 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2165 2166 unsigned OpIdx = I.getNumExplicitDefs(); 2167 if (!I.getOperand(OpIdx).isReg()) 2168 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2169 2170 // check if we may construct a constant vector 2171 Register OpReg = I.getOperand(OpIdx).getReg(); 2172 bool IsConst = isConstReg(MRI, OpReg); 2173 2174 if (!IsConst && N < 2) 2175 report_fatal_error( 2176 "There must be at least two constituent operands in a vector"); 2177 2178 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2179 TII.get(IsConst ? SPIRV::OpConstantComposite 2180 : SPIRV::OpCompositeConstruct)) 2181 .addDef(ResVReg) 2182 .addUse(GR.getSPIRVTypeID(ResType)); 2183 for (unsigned i = 0; i < N; ++i) 2184 MIB.addUse(OpReg); 2185 return MIB.constrainAllUses(TII, TRI, RBI); 2186 } 2187 2188 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2189 const SPIRVType *ResType, 2190 MachineInstr &I) const { 2191 2192 unsigned Opcode; 2193 2194 if (STI.canUseExtension( 2195 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2196 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2197 Opcode = SPIRV::OpDemoteToHelperInvocation; 2198 } else { 2199 Opcode = SPIRV::OpKill; 2200 // OpKill must be the last operation of any basic block. 2201 MachineInstr *NextI = I.getNextNode(); 2202 NextI->removeFromParent(); 2203 } 2204 2205 MachineBasicBlock &BB = *I.getParent(); 2206 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2207 .constrainAllUses(TII, TRI, RBI); 2208 } 2209 2210 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2211 const SPIRVType *ResType, 2212 unsigned CmpOpc, 2213 MachineInstr &I) const { 2214 Register Cmp0 = I.getOperand(2).getReg(); 2215 Register Cmp1 = I.getOperand(3).getReg(); 2216 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2217 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2218 "CMP operands should have the same type"); 2219 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2220 .addDef(ResVReg) 2221 .addUse(GR.getSPIRVTypeID(ResType)) 2222 .addUse(Cmp0) 2223 .addUse(Cmp1) 2224 .constrainAllUses(TII, TRI, RBI); 2225 } 2226 2227 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2228 const SPIRVType *ResType, 2229 MachineInstr &I) const { 2230 auto Pred = I.getOperand(1).getPredicate(); 2231 unsigned CmpOpc; 2232 2233 Register CmpOperand = I.getOperand(2).getReg(); 2234 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2235 CmpOpc = getPtrCmpOpcode(Pred); 2236 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2237 CmpOpc = getBoolCmpOpcode(Pred); 2238 else 2239 CmpOpc = getICmpOpcode(Pred); 2240 return selectCmp(ResVReg, ResType, CmpOpc, I); 2241 } 2242 2243 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2244 const MachineInstr &I, 2245 int OpIdx) const { 2246 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2247 "Expected G_FCONSTANT"); 2248 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2249 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2250 } 2251 2252 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2253 const MachineInstr &I, 2254 int OpIdx) const { 2255 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2256 "Expected G_CONSTANT"); 2257 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2258 } 2259 2260 std::pair<Register, bool> 2261 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2262 const SPIRVType *ResType) const { 2263 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2264 const SPIRVType *SpvI32Ty = 2265 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2266 // Find a constant in DT or build a new one. 2267 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2268 Register NewReg = GR.find(ConstInt, GR.CurMF); 2269 bool Result = true; 2270 if (!NewReg.isValid()) { 2271 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2272 GR.add(ConstInt, GR.CurMF, NewReg); 2273 MachineInstr *MI; 2274 MachineBasicBlock &BB = *I.getParent(); 2275 if (Val == 0) { 2276 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2277 .addDef(NewReg) 2278 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2279 } else { 2280 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2281 .addDef(NewReg) 2282 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2283 .addImm(APInt(32, Val).getZExtValue()); 2284 } 2285 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2286 } 2287 return {NewReg, Result}; 2288 } 2289 2290 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2291 const SPIRVType *ResType, 2292 MachineInstr &I) const { 2293 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2294 return selectCmp(ResVReg, ResType, CmpOp, I); 2295 } 2296 2297 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2298 MachineInstr &I) const { 2299 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2300 bool ZeroAsNull = STI.isOpenCLEnv(); 2301 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2302 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2303 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2304 } 2305 2306 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2307 MachineInstr &I) const { 2308 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2309 bool ZeroAsNull = STI.isOpenCLEnv(); 2310 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2311 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2312 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2313 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2314 } 2315 2316 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2317 MachineInstr &I) const { 2318 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2319 bool ZeroAsNull = STI.isOpenCLEnv(); 2320 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2321 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2322 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2323 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2324 } 2325 2326 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2327 const SPIRVType *ResType, 2328 MachineInstr &I) const { 2329 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2330 APInt One = 2331 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2332 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2333 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2334 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2335 } 2336 2337 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2338 const SPIRVType *ResType, 2339 MachineInstr &I, 2340 bool IsSigned) const { 2341 // To extend a bool, we need to use OpSelect between constants. 2342 Register ZeroReg = buildZerosVal(ResType, I); 2343 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2344 bool IsScalarBool = 2345 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2346 unsigned Opcode = 2347 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2348 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2349 .addDef(ResVReg) 2350 .addUse(GR.getSPIRVTypeID(ResType)) 2351 .addUse(I.getOperand(1).getReg()) 2352 .addUse(OneReg) 2353 .addUse(ZeroReg) 2354 .constrainAllUses(TII, TRI, RBI); 2355 } 2356 2357 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2358 const SPIRVType *ResType, 2359 MachineInstr &I, bool IsSigned, 2360 unsigned Opcode) const { 2361 Register SrcReg = I.getOperand(1).getReg(); 2362 // We can convert bool value directly to float type without OpConvert*ToF, 2363 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2364 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2365 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2366 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2367 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2368 const unsigned NumElts = ResType->getOperand(2).getImm(); 2369 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2370 } 2371 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2372 selectSelect(SrcReg, TmpType, I, false); 2373 } 2374 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2375 } 2376 2377 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2378 const SPIRVType *ResType, 2379 MachineInstr &I, bool IsSigned) const { 2380 Register SrcReg = I.getOperand(1).getReg(); 2381 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2382 return selectSelect(ResVReg, ResType, I, IsSigned); 2383 2384 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2385 if (SrcType == ResType) { 2386 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2387 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2388 if (DstRC != SrcRC && SrcRC) 2389 MRI->setRegClass(ResVReg, SrcRC); 2390 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2391 TII.get(TargetOpcode::COPY)) 2392 .addDef(ResVReg) 2393 .addUse(SrcReg) 2394 .constrainAllUses(TII, TRI, RBI); 2395 } 2396 2397 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2398 return selectUnOp(ResVReg, ResType, I, Opcode); 2399 } 2400 2401 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2402 Register ResVReg, 2403 MachineInstr &I, 2404 const SPIRVType *IntTy, 2405 const SPIRVType *BoolTy) const { 2406 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2407 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2408 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2409 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2410 Register Zero = buildZerosVal(IntTy, I); 2411 Register One = buildOnesVal(false, IntTy, I); 2412 MachineBasicBlock &BB = *I.getParent(); 2413 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2414 .addDef(BitIntReg) 2415 .addUse(GR.getSPIRVTypeID(IntTy)) 2416 .addUse(IntReg) 2417 .addUse(One) 2418 .constrainAllUses(TII, TRI, RBI); 2419 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2420 .addDef(ResVReg) 2421 .addUse(GR.getSPIRVTypeID(BoolTy)) 2422 .addUse(BitIntReg) 2423 .addUse(Zero) 2424 .constrainAllUses(TII, TRI, RBI); 2425 } 2426 2427 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2428 const SPIRVType *ResType, 2429 MachineInstr &I) const { 2430 Register IntReg = I.getOperand(1).getReg(); 2431 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2432 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2433 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2434 if (ArgType == ResType) { 2435 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2436 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2437 if (DstRC != SrcRC && SrcRC) 2438 MRI->setRegClass(ResVReg, SrcRC); 2439 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2440 TII.get(TargetOpcode::COPY)) 2441 .addDef(ResVReg) 2442 .addUse(IntReg) 2443 .constrainAllUses(TII, TRI, RBI); 2444 } 2445 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2446 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2447 return selectUnOp(ResVReg, ResType, I, Opcode); 2448 } 2449 2450 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2451 const SPIRVType *ResType, 2452 const APInt &Imm, 2453 MachineInstr &I) const { 2454 unsigned TyOpcode = ResType->getOpcode(); 2455 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2456 MachineBasicBlock &BB = *I.getParent(); 2457 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2458 Imm.isZero()) 2459 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2460 .addDef(ResVReg) 2461 .addUse(GR.getSPIRVTypeID(ResType)) 2462 .constrainAllUses(TII, TRI, RBI); 2463 if (TyOpcode == SPIRV::OpTypeInt) { 2464 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2465 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2466 if (Reg == ResVReg) 2467 return true; 2468 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2469 .addDef(ResVReg) 2470 .addUse(Reg) 2471 .constrainAllUses(TII, TRI, RBI); 2472 } 2473 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2474 .addDef(ResVReg) 2475 .addUse(GR.getSPIRVTypeID(ResType)); 2476 // <=32-bit integers should be caught by the sdag pattern. 2477 assert(Imm.getBitWidth() > 32); 2478 addNumImm(Imm, MIB); 2479 return MIB.constrainAllUses(TII, TRI, RBI); 2480 } 2481 2482 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2483 const SPIRVType *ResType, 2484 MachineInstr &I) const { 2485 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2486 .addDef(ResVReg) 2487 .addUse(GR.getSPIRVTypeID(ResType)) 2488 .constrainAllUses(TII, TRI, RBI); 2489 } 2490 2491 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2492 assert(MO.isReg()); 2493 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2494 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2495 assert(TypeInst->getOperand(1).isReg()); 2496 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2497 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2498 } 2499 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2500 } 2501 2502 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2503 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2504 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2505 return TypeInst->getOperand(2).getImm(); 2506 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2507 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2508 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2509 } 2510 2511 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2512 const SPIRVType *ResType, 2513 MachineInstr &I) const { 2514 MachineBasicBlock &BB = *I.getParent(); 2515 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2516 .addDef(ResVReg) 2517 .addUse(GR.getSPIRVTypeID(ResType)) 2518 // object to insert 2519 .addUse(I.getOperand(3).getReg()) 2520 // composite to insert into 2521 .addUse(I.getOperand(2).getReg()); 2522 for (unsigned i = 4; i < I.getNumOperands(); i++) 2523 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2524 return MIB.constrainAllUses(TII, TRI, RBI); 2525 } 2526 2527 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2528 const SPIRVType *ResType, 2529 MachineInstr &I) const { 2530 MachineBasicBlock &BB = *I.getParent(); 2531 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2532 .addDef(ResVReg) 2533 .addUse(GR.getSPIRVTypeID(ResType)) 2534 .addUse(I.getOperand(2).getReg()); 2535 for (unsigned i = 3; i < I.getNumOperands(); i++) 2536 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2537 return MIB.constrainAllUses(TII, TRI, RBI); 2538 } 2539 2540 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2541 const SPIRVType *ResType, 2542 MachineInstr &I) const { 2543 if (isImm(I.getOperand(4), MRI)) 2544 return selectInsertVal(ResVReg, ResType, I); 2545 MachineBasicBlock &BB = *I.getParent(); 2546 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2547 .addDef(ResVReg) 2548 .addUse(GR.getSPIRVTypeID(ResType)) 2549 .addUse(I.getOperand(2).getReg()) 2550 .addUse(I.getOperand(3).getReg()) 2551 .addUse(I.getOperand(4).getReg()) 2552 .constrainAllUses(TII, TRI, RBI); 2553 } 2554 2555 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2556 const SPIRVType *ResType, 2557 MachineInstr &I) const { 2558 if (isImm(I.getOperand(3), MRI)) 2559 return selectExtractVal(ResVReg, ResType, I); 2560 MachineBasicBlock &BB = *I.getParent(); 2561 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2562 .addDef(ResVReg) 2563 .addUse(GR.getSPIRVTypeID(ResType)) 2564 .addUse(I.getOperand(2).getReg()) 2565 .addUse(I.getOperand(3).getReg()) 2566 .constrainAllUses(TII, TRI, RBI); 2567 } 2568 2569 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2570 const SPIRVType *ResType, 2571 MachineInstr &I) const { 2572 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2573 2574 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2575 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2576 // we have to use Op[InBounds]AccessChain. 2577 const unsigned Opcode = STI.isVulkanEnv() 2578 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2579 : SPIRV::OpAccessChain) 2580 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2581 : SPIRV::OpPtrAccessChain); 2582 2583 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2584 .addDef(ResVReg) 2585 .addUse(GR.getSPIRVTypeID(ResType)) 2586 // Object to get a pointer to. 2587 .addUse(I.getOperand(3).getReg()); 2588 // Adding indices. 2589 const unsigned StartingIndex = 2590 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2591 ? 5 2592 : 4; 2593 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2594 Res.addUse(I.getOperand(i).getReg()); 2595 return Res.constrainAllUses(TII, TRI, RBI); 2596 } 2597 2598 // Maybe wrap a value into OpSpecConstantOp 2599 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2600 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2601 bool Result = true; 2602 unsigned Lim = I.getNumExplicitOperands(); 2603 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2604 Register OpReg = I.getOperand(i).getReg(); 2605 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2606 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2607 SmallPtrSet<SPIRVType *, 4> Visited; 2608 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2609 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2610 GR.isAggregateType(OpType)) { 2611 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2612 // by selectAddrSpaceCast() 2613 CompositeArgs.push_back(OpReg); 2614 continue; 2615 } 2616 MachineFunction *MF = I.getMF(); 2617 Register WrapReg = GR.find(OpDefine, MF); 2618 if (WrapReg.isValid()) { 2619 CompositeArgs.push_back(WrapReg); 2620 continue; 2621 } 2622 // Create a new register for the wrapper 2623 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2624 GR.add(OpDefine, MF, WrapReg); 2625 CompositeArgs.push_back(WrapReg); 2626 // Decorate the wrapper register and generate a new instruction 2627 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2628 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2629 MachineBasicBlock &BB = *I.getParent(); 2630 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2631 .addDef(WrapReg) 2632 .addUse(GR.getSPIRVTypeID(OpType)) 2633 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2634 .addUse(OpReg) 2635 .constrainAllUses(TII, TRI, RBI); 2636 if (!Result) 2637 break; 2638 } 2639 return Result; 2640 } 2641 2642 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2643 const SPIRVType *ResType, 2644 MachineInstr &I) const { 2645 MachineBasicBlock &BB = *I.getParent(); 2646 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2647 switch (IID) { 2648 case Intrinsic::spv_load: 2649 return selectLoad(ResVReg, ResType, I); 2650 case Intrinsic::spv_store: 2651 return selectStore(I); 2652 case Intrinsic::spv_extractv: 2653 return selectExtractVal(ResVReg, ResType, I); 2654 case Intrinsic::spv_insertv: 2655 return selectInsertVal(ResVReg, ResType, I); 2656 case Intrinsic::spv_extractelt: 2657 return selectExtractElt(ResVReg, ResType, I); 2658 case Intrinsic::spv_insertelt: 2659 return selectInsertElt(ResVReg, ResType, I); 2660 case Intrinsic::spv_gep: 2661 return selectGEP(ResVReg, ResType, I); 2662 case Intrinsic::spv_unref_global: 2663 case Intrinsic::spv_init_global: { 2664 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2665 MachineInstr *Init = I.getNumExplicitOperands() > 2 2666 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2667 : nullptr; 2668 assert(MI); 2669 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2670 } 2671 case Intrinsic::spv_undef: { 2672 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2673 .addDef(ResVReg) 2674 .addUse(GR.getSPIRVTypeID(ResType)); 2675 return MIB.constrainAllUses(TII, TRI, RBI); 2676 } 2677 case Intrinsic::spv_const_composite: { 2678 // If no values are attached, the composite is null constant. 2679 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2680 // Select a proper instruction. 2681 unsigned Opcode = SPIRV::OpConstantNull; 2682 SmallVector<Register> CompositeArgs; 2683 if (!IsNull) { 2684 Opcode = SPIRV::OpConstantComposite; 2685 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2686 return false; 2687 } 2688 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2689 .addDef(ResVReg) 2690 .addUse(GR.getSPIRVTypeID(ResType)); 2691 // skip type MD node we already used when generated assign.type for this 2692 if (!IsNull) { 2693 for (Register OpReg : CompositeArgs) 2694 MIB.addUse(OpReg); 2695 } 2696 return MIB.constrainAllUses(TII, TRI, RBI); 2697 } 2698 case Intrinsic::spv_assign_name: { 2699 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2700 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2701 for (unsigned i = I.getNumExplicitDefs() + 2; 2702 i < I.getNumExplicitOperands(); ++i) { 2703 MIB.addImm(I.getOperand(i).getImm()); 2704 } 2705 return MIB.constrainAllUses(TII, TRI, RBI); 2706 } 2707 case Intrinsic::spv_switch: { 2708 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2709 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2710 if (I.getOperand(i).isReg()) 2711 MIB.addReg(I.getOperand(i).getReg()); 2712 else if (I.getOperand(i).isCImm()) 2713 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2714 else if (I.getOperand(i).isMBB()) 2715 MIB.addMBB(I.getOperand(i).getMBB()); 2716 else 2717 llvm_unreachable("Unexpected OpSwitch operand"); 2718 } 2719 return MIB.constrainAllUses(TII, TRI, RBI); 2720 } 2721 case Intrinsic::spv_loop_merge: 2722 case Intrinsic::spv_selection_merge: { 2723 const auto Opcode = IID == Intrinsic::spv_selection_merge 2724 ? SPIRV::OpSelectionMerge 2725 : SPIRV::OpLoopMerge; 2726 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2727 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2728 assert(I.getOperand(i).isMBB()); 2729 MIB.addMBB(I.getOperand(i).getMBB()); 2730 } 2731 MIB.addImm(SPIRV::SelectionControl::None); 2732 return MIB.constrainAllUses(TII, TRI, RBI); 2733 } 2734 case Intrinsic::spv_cmpxchg: 2735 return selectAtomicCmpXchg(ResVReg, ResType, I); 2736 case Intrinsic::spv_unreachable: 2737 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2738 .constrainAllUses(TII, TRI, RBI); 2739 case Intrinsic::spv_alloca: 2740 return selectFrameIndex(ResVReg, ResType, I); 2741 case Intrinsic::spv_alloca_array: 2742 return selectAllocaArray(ResVReg, ResType, I); 2743 case Intrinsic::spv_assume: 2744 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2745 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2746 .addUse(I.getOperand(1).getReg()) 2747 .constrainAllUses(TII, TRI, RBI); 2748 break; 2749 case Intrinsic::spv_expect: 2750 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2751 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2752 .addDef(ResVReg) 2753 .addUse(GR.getSPIRVTypeID(ResType)) 2754 .addUse(I.getOperand(2).getReg()) 2755 .addUse(I.getOperand(3).getReg()) 2756 .constrainAllUses(TII, TRI, RBI); 2757 break; 2758 case Intrinsic::arithmetic_fence: 2759 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2760 return BuildMI(BB, I, I.getDebugLoc(), 2761 TII.get(SPIRV::OpArithmeticFenceEXT)) 2762 .addDef(ResVReg) 2763 .addUse(GR.getSPIRVTypeID(ResType)) 2764 .addUse(I.getOperand(2).getReg()) 2765 .constrainAllUses(TII, TRI, RBI); 2766 else 2767 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), 2768 ResVReg) 2769 .addUse(I.getOperand(2).getReg()) 2770 .constrainAllUses(TII, TRI, RBI); 2771 break; 2772 case Intrinsic::spv_thread_id: 2773 return selectSpvThreadId(ResVReg, ResType, I); 2774 case Intrinsic::spv_fdot: 2775 return selectFloatDot(ResVReg, ResType, I); 2776 case Intrinsic::spv_udot: 2777 case Intrinsic::spv_sdot: 2778 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2779 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2780 return selectIntegerDot(ResVReg, ResType, I, 2781 /*Signed=*/IID == Intrinsic::spv_sdot); 2782 return selectIntegerDotExpansion(ResVReg, ResType, I); 2783 case Intrinsic::spv_dot4add_i8packed: 2784 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2785 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2786 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2787 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2788 case Intrinsic::spv_dot4add_u8packed: 2789 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2790 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2791 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2792 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2793 case Intrinsic::spv_all: 2794 return selectAll(ResVReg, ResType, I); 2795 case Intrinsic::spv_any: 2796 return selectAny(ResVReg, ResType, I); 2797 case Intrinsic::spv_cross: 2798 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2799 case Intrinsic::spv_lerp: 2800 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2801 case Intrinsic::spv_length: 2802 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2803 case Intrinsic::spv_degrees: 2804 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2805 case Intrinsic::spv_frac: 2806 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2807 case Intrinsic::spv_normalize: 2808 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2809 case Intrinsic::spv_rsqrt: 2810 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2811 case Intrinsic::spv_sign: 2812 return selectSign(ResVReg, ResType, I); 2813 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2814 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2815 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2816 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2817 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2818 bool Result = true; 2819 auto MemSemConstant = 2820 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2821 Register MemSemReg = MemSemConstant.first; 2822 Result &= MemSemConstant.second; 2823 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2824 Register ScopeReg = ScopeConstant.first; 2825 Result &= ScopeConstant.second; 2826 MachineBasicBlock &BB = *I.getParent(); 2827 return Result && 2828 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2829 .addUse(ScopeReg) 2830 .addUse(ScopeReg) 2831 .addUse(MemSemReg) 2832 .constrainAllUses(TII, TRI, RBI); 2833 } 2834 case Intrinsic::spv_lifetime_start: 2835 case Intrinsic::spv_lifetime_end: { 2836 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2837 : SPIRV::OpLifetimeStop; 2838 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2839 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2840 if (Size == -1) 2841 Size = 0; 2842 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2843 .addUse(PtrReg) 2844 .addImm(Size) 2845 .constrainAllUses(TII, TRI, RBI); 2846 } 2847 case Intrinsic::spv_saturate: 2848 return selectSaturate(ResVReg, ResType, I); 2849 case Intrinsic::spv_nclamp: 2850 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2851 case Intrinsic::spv_uclamp: 2852 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2853 case Intrinsic::spv_sclamp: 2854 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2855 case Intrinsic::spv_wave_active_countbits: 2856 return selectWaveActiveCountBits(ResVReg, ResType, I); 2857 case Intrinsic::spv_wave_any: 2858 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 2859 case Intrinsic::spv_wave_is_first_lane: 2860 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 2861 case Intrinsic::spv_wave_readlane: 2862 return selectWaveOpInst(ResVReg, ResType, I, 2863 SPIRV::OpGroupNonUniformShuffle); 2864 case Intrinsic::spv_step: 2865 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2866 case Intrinsic::spv_radians: 2867 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2868 // Discard intrinsics which we do not expect to actually represent code after 2869 // lowering or intrinsics which are not implemented but should not crash when 2870 // found in a customer's LLVM IR input. 2871 case Intrinsic::instrprof_increment: 2872 case Intrinsic::instrprof_increment_step: 2873 case Intrinsic::instrprof_value_profile: 2874 break; 2875 // Discard internal intrinsics. 2876 case Intrinsic::spv_value_md: 2877 break; 2878 case Intrinsic::spv_handle_fromBinding: { 2879 return selectHandleFromBinding(ResVReg, ResType, I); 2880 } 2881 case Intrinsic::spv_typedBufferStore: { 2882 selectImageWriteIntrinsic(I); 2883 return true; 2884 } 2885 case Intrinsic::spv_typedBufferLoad: { 2886 selectReadImageIntrinsic(ResVReg, ResType, I); 2887 return true; 2888 } 2889 case Intrinsic::spv_discard: { 2890 return selectDiscard(ResVReg, ResType, I); 2891 } 2892 default: { 2893 std::string DiagMsg; 2894 raw_string_ostream OS(DiagMsg); 2895 I.print(OS); 2896 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2897 report_fatal_error(DiagMsg.c_str(), false); 2898 } 2899 } 2900 return true; 2901 } 2902 2903 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2904 const SPIRVType *ResType, 2905 MachineInstr &I) const { 2906 2907 uint32_t Set = foldImm(I.getOperand(2), MRI); 2908 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2909 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2910 Register IndexReg = I.getOperand(5).getReg(); 2911 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2912 2913 MachineIRBuilder MIRBuilder(I); 2914 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2915 IndexReg, IsNonUniform, MIRBuilder); 2916 2917 if (IsNonUniform) 2918 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2919 2920 // TODO: For now we assume the resource is an image, which needs to be 2921 // loaded to get the handle. That will not be true for storage buffers. 2922 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2923 .addDef(ResVReg) 2924 .addUse(GR.getSPIRVTypeID(ResType)) 2925 .addUse(VarReg) 2926 .constrainAllUses(TII, TRI, RBI); 2927 } 2928 2929 void SPIRVInstructionSelector::selectReadImageIntrinsic( 2930 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2931 2932 // If the load of the image is in a different basic block, then 2933 // this will generate invalid code. A proper solution is to move 2934 // the OpLoad from selectHandleFromBinding here. However, to do 2935 // that we will need to change the return type of the intrinsic. 2936 // We will do that when we can, but for now trying to move forward with other 2937 // issues. 2938 Register ImageReg = I.getOperand(2).getReg(); 2939 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 2940 "The image must be loaded in the same basic block as its use."); 2941 2942 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2943 if (ResultSize == 4) { 2944 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 2945 .addDef(ResVReg) 2946 .addUse(GR.getSPIRVTypeID(ResType)) 2947 .addUse(ImageReg) 2948 .addUse(I.getOperand(3).getReg()); 2949 return; 2950 } 2951 2952 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 2953 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 2954 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 2955 .addDef(ReadReg) 2956 .addUse(GR.getSPIRVTypeID(ReadType)) 2957 .addUse(ImageReg) 2958 .addUse(I.getOperand(3).getReg()); 2959 2960 if (ResultSize == 1) { 2961 BuildMI(*I.getParent(), I, I.getDebugLoc(), 2962 TII.get(SPIRV::OpCompositeExtract)) 2963 .addDef(ResVReg) 2964 .addUse(GR.getSPIRVTypeID(ResType)) 2965 .addUse(ReadReg) 2966 .addImm(0); 2967 return; 2968 } 2969 extractSubvector(ResVReg, ResType, ReadReg, I); 2970 } 2971 2972 void SPIRVInstructionSelector::extractSubvector( 2973 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 2974 MachineInstr &InsertionPoint) const { 2975 SPIRVType *InputType = GR.getResultType(ReadReg); 2976 [[maybe_unused]] uint64_t InputSize = 2977 GR.getScalarOrVectorComponentCount(InputType); 2978 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2979 assert(InputSize > 1 && "The input must be a vector."); 2980 assert(ResultSize > 1 && "The result must be a vector."); 2981 assert(ResultSize < InputSize && 2982 "Cannot extract more element than there are in the input."); 2983 SmallVector<Register> ComponentRegisters; 2984 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 2985 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 2986 for (uint64_t I = 0; I < ResultSize; I++) { 2987 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 2988 BuildMI(*InsertionPoint.getParent(), InsertionPoint, 2989 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2990 .addDef(ComponentReg) 2991 .addUse(ScalarType->getOperand(0).getReg()) 2992 .addUse(ReadReg) 2993 .addImm(I); 2994 ComponentRegisters.emplace_back(ComponentReg); 2995 } 2996 2997 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 2998 InsertionPoint.getDebugLoc(), 2999 TII.get(SPIRV::OpCompositeConstruct)) 3000 .addDef(ResVReg) 3001 .addUse(GR.getSPIRVTypeID(ResType)); 3002 3003 for (Register ComponentReg : ComponentRegisters) 3004 MIB.addUse(ComponentReg); 3005 } 3006 3007 void SPIRVInstructionSelector::selectImageWriteIntrinsic( 3008 MachineInstr &I) const { 3009 // If the load of the image is in a different basic block, then 3010 // this will generate invalid code. A proper solution is to move 3011 // the OpLoad from selectHandleFromBinding here. However, to do 3012 // that we will need to change the return type of the intrinsic. 3013 // We will do that when we can, but for now trying to move forward with other 3014 // issues. 3015 Register ImageReg = I.getOperand(1).getReg(); 3016 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3017 "The image must be loaded in the same basic block as its use."); 3018 Register CoordinateReg = I.getOperand(2).getReg(); 3019 Register DataReg = I.getOperand(3).getReg(); 3020 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3021 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3022 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite)) 3023 .addUse(ImageReg) 3024 .addUse(CoordinateReg) 3025 .addUse(DataReg); 3026 } 3027 3028 Register SPIRVInstructionSelector::buildPointerToResource( 3029 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3030 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3031 MachineIRBuilder MIRBuilder) const { 3032 if (ArraySize == 1) 3033 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3034 MIRBuilder); 3035 3036 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3037 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3038 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3039 VarType, Set, Binding, MIRBuilder); 3040 3041 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3042 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3043 3044 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3045 if (IsNonUniform) { 3046 // It is unclear which value needs to be marked an non-uniform, so both 3047 // the index and the access changed are decorated as non-uniform. 3048 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3049 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3050 } 3051 3052 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3053 .addDef(AcReg) 3054 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3055 .addUse(VarReg) 3056 .addUse(IndexReg); 3057 3058 return AcReg; 3059 } 3060 3061 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3062 const SPIRVType *ResType, 3063 MachineInstr &I, 3064 bool IsSigned) const { 3065 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3066 // zero or sign extend 3067 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3068 bool Result = 3069 selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, Opcode); 3070 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3071 } 3072 3073 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3074 const SPIRVType *ResType, 3075 MachineInstr &I, 3076 Register SrcReg, 3077 bool IsSigned) const { 3078 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3079 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3080 .addDef(ResVReg) 3081 .addUse(GR.getSPIRVTypeID(ResType)) 3082 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3083 .addImm(Opcode) 3084 .addUse(SrcReg) 3085 .constrainAllUses(TII, TRI, RBI); 3086 } 3087 3088 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3089 const SPIRVType *ResType, 3090 MachineInstr &I, 3091 bool IsSigned) const { 3092 Register OpReg = I.getOperand(2).getReg(); 3093 // 1. split our int64 into 2 pieces using a bitcast 3094 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3095 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3096 MachineIRBuilder MIRBuilder(I); 3097 SPIRVType *postCastT = 3098 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3099 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3100 bool Result = 3101 selectOpWithSrcs(bitcastReg, postCastT, I, {OpReg}, SPIRV::OpBitcast); 3102 3103 // 2. call firstbithigh 3104 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3105 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3106 3107 // 3. split result vector into high bits and low bits 3108 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3109 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3110 3111 bool ZeroAsNull = STI.isOpenCLEnv(); 3112 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3113 if (isScalarRes) { 3114 // if scalar do a vector extract 3115 Result &= selectOpWithSrcs( 3116 HighReg, ResType, I, 3117 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3118 SPIRV::OpVectorExtractDynamic); 3119 Result &= selectOpWithSrcs( 3120 LowReg, ResType, I, 3121 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3122 SPIRV::OpVectorExtractDynamic); 3123 } else { // vector case do a shufflevector 3124 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3125 TII.get(SPIRV::OpVectorShuffle)) 3126 .addDef(HighReg) 3127 .addUse(GR.getSPIRVTypeID(ResType)) 3128 .addUse(FBHReg) 3129 .addUse(FBHReg); 3130 // ^^ this vector will not be selected from; could be empty 3131 unsigned j; 3132 for (j = 0; j < count * 2; j += 2) { 3133 MIB.addImm(j); 3134 } 3135 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3136 3137 // get low bits 3138 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3139 TII.get(SPIRV::OpVectorShuffle)) 3140 .addDef(LowReg) 3141 .addUse(GR.getSPIRVTypeID(ResType)) 3142 .addUse(FBHReg) 3143 .addUse(FBHReg); 3144 // ^^ this vector will not be selected from; could be empty 3145 for (j = 1; j < count * 2; j += 2) { 3146 MIB.addImm(j); 3147 } 3148 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3149 } 3150 3151 // 4. check if result of each top 32 bits is == -1 3152 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3153 Register NegOneReg; 3154 Register Reg0; 3155 Register Reg32; 3156 unsigned selectOp; 3157 unsigned addOp; 3158 if (isScalarRes) { 3159 NegOneReg = 3160 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3161 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3162 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3163 selectOp = SPIRV::OpSelectSISCond; 3164 addOp = SPIRV::OpIAddS; 3165 } else { 3166 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3167 NegOneReg = 3168 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3169 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3170 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3171 selectOp = SPIRV::OpSelectVIVCond; 3172 addOp = SPIRV::OpIAddV; 3173 } 3174 3175 // check if the high bits are == -1; true if -1 3176 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3177 Result &= selectOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3178 SPIRV::OpIEqual); 3179 3180 // Select low bits if true in BReg, otherwise high bits 3181 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3182 Result &= 3183 selectOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, selectOp); 3184 3185 // Add 32 for high bits, 0 for low bits 3186 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3187 Result &= selectOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3188 3189 return Result && 3190 selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3191 } 3192 3193 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3194 const SPIRVType *ResType, 3195 MachineInstr &I, 3196 bool IsSigned) const { 3197 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3198 Register OpReg = I.getOperand(2).getReg(); 3199 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3200 3201 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3202 case 16: 3203 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3204 case 32: 3205 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3206 case 64: 3207 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3208 default: 3209 report_fatal_error( 3210 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3211 } 3212 } 3213 3214 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3215 const SPIRVType *ResType, 3216 MachineInstr &I) const { 3217 // there was an allocation size parameter to the allocation instruction 3218 // that is not 1 3219 MachineBasicBlock &BB = *I.getParent(); 3220 return BuildMI(BB, I, I.getDebugLoc(), 3221 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3222 .addDef(ResVReg) 3223 .addUse(GR.getSPIRVTypeID(ResType)) 3224 .addUse(I.getOperand(2).getReg()) 3225 .constrainAllUses(TII, TRI, RBI); 3226 } 3227 3228 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3229 const SPIRVType *ResType, 3230 MachineInstr &I) const { 3231 // Change order of instructions if needed: all OpVariable instructions in a 3232 // function must be the first instructions in the first block 3233 auto It = getOpVariableMBBIt(I); 3234 return BuildMI(*It->getParent(), It, It->getDebugLoc(), 3235 TII.get(SPIRV::OpVariable)) 3236 .addDef(ResVReg) 3237 .addUse(GR.getSPIRVTypeID(ResType)) 3238 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3239 .constrainAllUses(TII, TRI, RBI); 3240 } 3241 3242 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3243 // InstructionSelector walks backwards through the instructions. We can use 3244 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3245 // first, so can generate an OpBranchConditional here. If there is no 3246 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3247 const MachineInstr *PrevI = I.getPrevNode(); 3248 MachineBasicBlock &MBB = *I.getParent(); 3249 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3250 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3251 .addUse(PrevI->getOperand(0).getReg()) 3252 .addMBB(PrevI->getOperand(1).getMBB()) 3253 .addMBB(I.getOperand(0).getMBB()) 3254 .constrainAllUses(TII, TRI, RBI); 3255 } 3256 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3257 .addMBB(I.getOperand(0).getMBB()) 3258 .constrainAllUses(TII, TRI, RBI); 3259 } 3260 3261 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3262 // InstructionSelector walks backwards through the instructions. For an 3263 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3264 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3265 // generate the OpBranchConditional in selectBranch above. 3266 // 3267 // If an OpBranchConditional has been generated, we simply return, as the work 3268 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3269 // implicit fallthrough to the next basic block, so we need to create an 3270 // OpBranchConditional with an explicit "false" argument pointing to the next 3271 // basic block that LLVM would fall through to. 3272 const MachineInstr *NextI = I.getNextNode(); 3273 // Check if this has already been successfully selected. 3274 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3275 return true; 3276 // Must be relying on implicit block fallthrough, so generate an 3277 // OpBranchConditional with the "next" basic block as the "false" target. 3278 MachineBasicBlock &MBB = *I.getParent(); 3279 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3280 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3281 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3282 .addUse(I.getOperand(0).getReg()) 3283 .addMBB(I.getOperand(1).getMBB()) 3284 .addMBB(NextMBB) 3285 .constrainAllUses(TII, TRI, RBI); 3286 } 3287 3288 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3289 const SPIRVType *ResType, 3290 MachineInstr &I) const { 3291 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3292 .addDef(ResVReg) 3293 .addUse(GR.getSPIRVTypeID(ResType)); 3294 const unsigned NumOps = I.getNumOperands(); 3295 for (unsigned i = 1; i < NumOps; i += 2) { 3296 MIB.addUse(I.getOperand(i + 0).getReg()); 3297 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3298 } 3299 return MIB.constrainAllUses(TII, TRI, RBI); 3300 } 3301 3302 bool SPIRVInstructionSelector::selectGlobalValue( 3303 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3304 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3305 MachineIRBuilder MIRBuilder(I); 3306 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3307 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3308 SPIRVType *PointerBaseType; 3309 if (GVType->isArrayTy()) { 3310 SPIRVType *ArrayElementType = 3311 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3312 SPIRV::AccessQualifier::ReadWrite, false); 3313 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3314 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3315 } else { 3316 PointerBaseType = GR.getOrCreateSPIRVType( 3317 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3318 } 3319 3320 std::string GlobalIdent; 3321 if (!GV->hasName()) { 3322 unsigned &ID = UnnamedGlobalIDs[GV]; 3323 if (ID == 0) 3324 ID = UnnamedGlobalIDs.size(); 3325 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3326 } else { 3327 GlobalIdent = GV->getGlobalIdentifier(); 3328 } 3329 3330 // Behaviour of functions as operands depends on availability of the 3331 // corresponding extension (SPV_INTEL_function_pointers): 3332 // - If there is an extension to operate with functions as operands: 3333 // We create a proper constant operand and evaluate a correct type for a 3334 // function pointer. 3335 // - Without the required extension: 3336 // We have functions as operands in tests with blocks of instruction e.g. in 3337 // transcoding/global_block.ll. These operands are not used and should be 3338 // substituted by zero constants. Their type is expected to be always 3339 // OpTypePointer Function %uchar. 3340 if (isa<Function>(GV)) { 3341 const Constant *ConstVal = GV; 3342 MachineBasicBlock &BB = *I.getParent(); 3343 Register NewReg = GR.find(ConstVal, GR.CurMF); 3344 if (!NewReg.isValid()) { 3345 Register NewReg = ResVReg; 3346 GR.add(ConstVal, GR.CurMF, NewReg); 3347 const Function *GVFun = 3348 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3349 ? dyn_cast<Function>(GV) 3350 : nullptr; 3351 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3352 PointerBaseType, I, TII, 3353 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3354 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3355 if (GVFun) { 3356 // References to a function via function pointers generate virtual 3357 // registers without a definition. We will resolve it later, during 3358 // module analysis stage. 3359 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3360 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3361 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3362 MachineInstrBuilder MB = 3363 BuildMI(BB, I, I.getDebugLoc(), 3364 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3365 .addDef(NewReg) 3366 .addUse(GR.getSPIRVTypeID(ResType)) 3367 .addUse(FuncVReg); 3368 // mapping the function pointer to the used Function 3369 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3370 return MB.constrainAllUses(TII, TRI, RBI); 3371 } 3372 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3373 .addDef(NewReg) 3374 .addUse(GR.getSPIRVTypeID(ResType)) 3375 .constrainAllUses(TII, TRI, RBI); 3376 } 3377 assert(NewReg != ResVReg); 3378 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 3379 .addDef(ResVReg) 3380 .addUse(NewReg) 3381 .constrainAllUses(TII, TRI, RBI); 3382 } 3383 auto GlobalVar = cast<GlobalVariable>(GV); 3384 assert(GlobalVar->getName() != "llvm.global.annotations"); 3385 3386 bool HasInit = GlobalVar->hasInitializer() && 3387 !isa<UndefValue>(GlobalVar->getInitializer()); 3388 // Skip empty declaration for GVs with initilaizers till we get the decl with 3389 // passed initializer. 3390 if (HasInit && !Init) 3391 return true; 3392 3393 unsigned AddrSpace = GV->getAddressSpace(); 3394 SPIRV::StorageClass::StorageClass Storage = 3395 addressSpaceToStorageClass(AddrSpace, STI); 3396 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 3397 Storage != SPIRV::StorageClass::Function; 3398 SPIRV::LinkageType::LinkageType LnkType = 3399 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3400 ? SPIRV::LinkageType::Import 3401 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3402 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3403 ? SPIRV::LinkageType::LinkOnceODR 3404 : SPIRV::LinkageType::Export); 3405 3406 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3407 PointerBaseType, I, TII, 3408 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3409 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 3410 Storage, Init, GlobalVar->isConstant(), 3411 HasLnkTy, LnkType, MIRBuilder, true); 3412 return Reg.isValid(); 3413 } 3414 3415 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3416 const SPIRVType *ResType, 3417 MachineInstr &I) const { 3418 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3419 return selectExtInst(ResVReg, ResType, I, CL::log10); 3420 } 3421 3422 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3423 // is implemented as: 3424 // log10(x) = log2(x) * (1 / log2(10)) 3425 // = log2(x) * 0.30103 3426 3427 MachineIRBuilder MIRBuilder(I); 3428 MachineBasicBlock &BB = *I.getParent(); 3429 3430 // Build log2(x). 3431 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3432 bool Result = 3433 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3434 .addDef(VarReg) 3435 .addUse(GR.getSPIRVTypeID(ResType)) 3436 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3437 .addImm(GL::Log2) 3438 .add(I.getOperand(1)) 3439 .constrainAllUses(TII, TRI, RBI); 3440 3441 // Build 0.30103. 3442 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3443 ResType->getOpcode() == SPIRV::OpTypeFloat); 3444 // TODO: Add matrix implementation once supported by the HLSL frontend. 3445 const SPIRVType *SpirvScalarType = 3446 ResType->getOpcode() == SPIRV::OpTypeVector 3447 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3448 : ResType; 3449 Register ScaleReg = 3450 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3451 3452 // Multiply log2(x) by 0.30103 to get log10(x) result. 3453 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3454 ? SPIRV::OpVectorTimesScalar 3455 : SPIRV::OpFMulS; 3456 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3457 .addDef(ResVReg) 3458 .addUse(GR.getSPIRVTypeID(ResType)) 3459 .addUse(VarReg) 3460 .addUse(ScaleReg) 3461 .constrainAllUses(TII, TRI, RBI); 3462 } 3463 3464 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3465 const SPIRVType *ResType, 3466 MachineInstr &I) const { 3467 // DX intrinsic: @llvm.dx.thread.id(i32) 3468 // ID Name Description 3469 // 93 ThreadId reads the thread ID 3470 3471 MachineIRBuilder MIRBuilder(I); 3472 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3473 const SPIRVType *Vec3Ty = 3474 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3475 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3476 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3477 3478 // Create new register for GlobalInvocationID builtin variable. 3479 Register NewRegister = 3480 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3481 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3482 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3483 3484 // Build GlobalInvocationID global variable with the necessary decorations. 3485 Register Variable = GR.buildGlobalVariable( 3486 NewRegister, PtrType, 3487 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3488 SPIRV::StorageClass::Input, nullptr, true, true, 3489 SPIRV::LinkageType::Import, MIRBuilder, false); 3490 3491 // Create new register for loading value. 3492 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3493 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3494 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3495 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3496 3497 // Load v3uint value from the global variable. 3498 bool Result = 3499 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3500 .addDef(LoadedRegister) 3501 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3502 .addUse(Variable); 3503 3504 // Get Thread ID index. Expecting operand is a constant immediate value, 3505 // wrapped in a type assignment. 3506 assert(I.getOperand(2).isReg()); 3507 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3508 3509 // Extract the thread ID from the loaded vector value. 3510 MachineBasicBlock &BB = *I.getParent(); 3511 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3512 .addDef(ResVReg) 3513 .addUse(GR.getSPIRVTypeID(ResType)) 3514 .addUse(LoadedRegister) 3515 .addImm(ThreadId); 3516 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3517 } 3518 3519 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3520 MachineInstr &I) const { 3521 MachineIRBuilder MIRBuilder(I); 3522 if (Type->getOpcode() != SPIRV::OpTypeVector) 3523 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3524 3525 uint64_t VectorSize = Type->getOperand(2).getImm(); 3526 if (VectorSize == 4) 3527 return Type; 3528 3529 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3530 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3531 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3532 } 3533 3534 namespace llvm { 3535 InstructionSelector * 3536 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3537 const SPIRVSubtarget &Subtarget, 3538 const RegisterBankInfo &RBI) { 3539 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3540 } 3541 } // namespace llvm 3542