1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 37 #define DEBUG_TYPE "spirv-isel" 38 39 using namespace llvm; 40 namespace CL = SPIRV::OpenCLExtInst; 41 namespace GL = SPIRV::GLSLExtInst; 42 43 using ExtInstList = 44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 45 46 namespace { 47 48 #define GET_GLOBALISEL_PREDICATE_BITSET 49 #include "SPIRVGenGlobalISel.inc" 50 #undef GET_GLOBALISEL_PREDICATE_BITSET 51 52 class SPIRVInstructionSelector : public InstructionSelector { 53 const SPIRVSubtarget &STI; 54 const SPIRVInstrInfo &TII; 55 const SPIRVRegisterInfo &TRI; 56 const RegisterBankInfo &RBI; 57 SPIRVGlobalRegistry &GR; 58 MachineRegisterInfo *MRI; 59 MachineFunction *HasVRegsReset = nullptr; 60 61 /// We need to keep track of the number we give to anonymous global values to 62 /// generate the same name every time when this is needed. 63 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 64 65 public: 66 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 67 const SPIRVSubtarget &ST, 68 const RegisterBankInfo &RBI); 69 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 70 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 71 BlockFrequencyInfo *BFI) override; 72 // Common selection code. Instruction-specific selection occurs in spvSelect. 73 bool select(MachineInstr &I) override; 74 static const char *getName() { return DEBUG_TYPE; } 75 76 #define GET_GLOBALISEL_PREDICATES_DECL 77 #include "SPIRVGenGlobalISel.inc" 78 #undef GET_GLOBALISEL_PREDICATES_DECL 79 80 #define GET_GLOBALISEL_TEMPORARIES_DECL 81 #include "SPIRVGenGlobalISel.inc" 82 #undef GET_GLOBALISEL_TEMPORARIES_DECL 83 84 private: 85 void resetVRegsType(MachineFunction &MF); 86 87 // tblgen-erated 'select' implementation, used as the initial selector for 88 // the patterns that don't require complex C++. 89 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 90 91 // All instruction-specific selection that didn't happen in "select()". 92 // Is basically a large Switch/Case delegating to all other select method. 93 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 94 MachineInstr &I) const; 95 96 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 97 MachineInstr &I, bool IsSigned) const; 98 99 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 100 MachineInstr &I, bool IsSigned) const; 101 102 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 103 MachineInstr &I, Register SrcReg, 104 bool IsSigned) const; 105 106 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I, bool IsSigned) const; 108 109 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 110 const MachineInstr *Init = nullptr) const; 111 112 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I, std::vector<Register> SrcRegs, 114 unsigned Opcode) const; 115 116 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 117 unsigned Opcode) const; 118 119 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I) const; 121 122 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 123 MachineInstr &I) const; 124 bool selectStore(MachineInstr &I) const; 125 126 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 bool selectStackRestore(MachineInstr &I) const; 129 130 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 131 132 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I, unsigned NewOpcode, 134 unsigned NegateOpcode = 0) const; 135 136 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 137 MachineInstr &I) const; 138 139 bool selectFence(MachineInstr &I) const; 140 141 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 142 MachineInstr &I) const; 143 144 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 145 MachineInstr &I, unsigned OpType) const; 146 147 bool selectAll(Register ResVReg, const SPIRVType *ResType, 148 MachineInstr &I) const; 149 150 bool selectAny(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 153 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 154 MachineInstr &I) const; 155 156 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 157 MachineInstr &I) const; 158 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 162 unsigned comparisonOpcode, MachineInstr &I) const; 163 bool selectCross(Register ResVReg, const SPIRVType *ResType, 164 MachineInstr &I) const; 165 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 168 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectSign(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I, unsigned Opcode) const; 181 182 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I, bool Signed) const; 184 185 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 188 template <bool Signed> 189 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 190 MachineInstr &I) const; 191 template <bool Signed> 192 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 195 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 196 int OpIdx) const; 197 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 198 int OpIdx) const; 199 200 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 201 MachineInstr &I) const; 202 203 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 204 bool IsSigned) const; 205 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 206 bool IsSigned, unsigned Opcode) const; 207 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 208 bool IsSigned) const; 209 210 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 211 MachineInstr &I) const; 212 213 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 214 bool IsSigned) const; 215 216 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 217 const SPIRVType *intTy, const SPIRVType *boolTy) const; 218 219 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 220 MachineInstr &I) const; 221 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I) const; 223 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 224 MachineInstr &I) const; 225 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 226 MachineInstr &I) const; 227 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I) const; 229 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 230 MachineInstr &I) const; 231 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 232 MachineInstr &I) const; 233 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 234 MachineInstr &I) const; 235 236 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 237 MachineInstr &I) const; 238 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 239 MachineInstr &I) const; 240 241 bool selectBranch(MachineInstr &I) const; 242 bool selectBranchCond(MachineInstr &I) const; 243 244 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 245 MachineInstr &I) const; 246 247 [[maybe_unused]] bool selectExtInst(Register ResVReg, 248 const SPIRVType *RestType, 249 MachineInstr &I, 250 GL::GLSLExtInst GLInst) const; 251 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 253 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 254 MachineInstr &I, CL::OpenCLExtInst CLInst, 255 GL::GLSLExtInst GLInst) const; 256 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 257 MachineInstr &I, const ExtInstList &ExtInsts) const; 258 259 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 262 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 263 MachineInstr &I) const; 264 265 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 266 MachineInstr &I) const; 267 268 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 269 MachineInstr &I, unsigned Opcode) const; 270 271 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 272 MachineInstr &I) const; 273 274 bool selectUnmergeValues(MachineInstr &I) const; 275 276 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 277 MachineInstr &I) const; 278 279 void selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 280 MachineInstr &I) const; 281 282 void selectImageWriteIntrinsic(MachineInstr &I) const; 283 284 // Utilities 285 std::pair<Register, bool> 286 buildI32Constant(uint32_t Val, MachineInstr &I, 287 const SPIRVType *ResType = nullptr) const; 288 289 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 290 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 291 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 292 MachineInstr &I) const; 293 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 294 295 bool wrapIntoSpecConstantOp(MachineInstr &I, 296 SmallVector<Register> &CompositeArgs) const; 297 298 Register getUcharPtrTypeReg(MachineInstr &I, 299 SPIRV::StorageClass::StorageClass SC) const; 300 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 301 Register Src, Register DestType, 302 uint32_t Opcode) const; 303 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 304 SPIRVType *SrcPtrTy) const; 305 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 306 uint32_t Binding, uint32_t ArraySize, 307 Register IndexReg, bool IsNonUniform, 308 MachineIRBuilder MIRBuilder) const; 309 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 310 void extractSubvector(Register &ResVReg, const SPIRVType *ResType, 311 Register &ReadReg, MachineInstr &InsertionPoint) const; 312 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 313 }; 314 315 } // end anonymous namespace 316 317 #define GET_GLOBALISEL_IMPL 318 #include "SPIRVGenGlobalISel.inc" 319 #undef GET_GLOBALISEL_IMPL 320 321 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 322 const SPIRVSubtarget &ST, 323 const RegisterBankInfo &RBI) 324 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 325 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 326 #define GET_GLOBALISEL_PREDICATES_INIT 327 #include "SPIRVGenGlobalISel.inc" 328 #undef GET_GLOBALISEL_PREDICATES_INIT 329 #define GET_GLOBALISEL_TEMPORARIES_INIT 330 #include "SPIRVGenGlobalISel.inc" 331 #undef GET_GLOBALISEL_TEMPORARIES_INIT 332 { 333 } 334 335 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 336 CodeGenCoverage *CoverageInfo, 337 ProfileSummaryInfo *PSI, 338 BlockFrequencyInfo *BFI) { 339 MRI = &MF.getRegInfo(); 340 GR.setCurrentFunc(MF); 341 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 342 } 343 344 // Ensure that register classes correspond to pattern matching rules. 345 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 346 if (HasVRegsReset == &MF) 347 return; 348 HasVRegsReset = &MF; 349 350 MachineRegisterInfo &MRI = MF.getRegInfo(); 351 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 352 Register Reg = Register::index2VirtReg(I); 353 LLT RegType = MRI.getType(Reg); 354 if (RegType.isScalar()) 355 MRI.setType(Reg, LLT::scalar(64)); 356 else if (RegType.isPointer()) 357 MRI.setType(Reg, LLT::pointer(0, 64)); 358 else if (RegType.isVector()) 359 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 360 } 361 for (const auto &MBB : MF) { 362 for (const auto &MI : MBB) { 363 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 364 continue; 365 Register DstReg = MI.getOperand(0).getReg(); 366 LLT DstType = MRI.getType(DstReg); 367 Register SrcReg = MI.getOperand(1).getReg(); 368 LLT SrcType = MRI.getType(SrcReg); 369 if (DstType != SrcType) 370 MRI.setType(DstReg, MRI.getType(SrcReg)); 371 372 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 373 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 374 if (DstRC != SrcRC && SrcRC) 375 MRI.setRegClass(DstReg, SrcRC); 376 } 377 } 378 } 379 380 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 381 382 // Defined in SPIRVLegalizerInfo.cpp. 383 extern bool isTypeFoldingSupported(unsigned Opcode); 384 385 bool SPIRVInstructionSelector::select(MachineInstr &I) { 386 resetVRegsType(*I.getParent()->getParent()); 387 388 assert(I.getParent() && "Instruction should be in a basic block!"); 389 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 390 391 Register Opcode = I.getOpcode(); 392 // If it's not a GMIR instruction, we've selected it already. 393 if (!isPreISelGenericOpcode(Opcode)) { 394 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 395 Register DstReg = I.getOperand(0).getReg(); 396 Register SrcReg = I.getOperand(1).getReg(); 397 auto *Def = MRI->getVRegDef(SrcReg); 398 if (isTypeFoldingSupported(Def->getOpcode())) { 399 bool Res = selectImpl(I, *CoverageInfo); 400 LLVM_DEBUG({ 401 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 402 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 403 I.print(dbgs()); 404 } 405 }); 406 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 407 if (Res) 408 return Res; 409 } 410 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 411 MRI->replaceRegWith(SrcReg, DstReg); 412 I.removeFromParent(); 413 return true; 414 } else if (I.getNumDefs() == 1) { 415 // Make all vregs 64 bits (for SPIR-V IDs). 416 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 417 } 418 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 419 } 420 421 if (I.getNumOperands() != I.getNumExplicitOperands()) { 422 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 423 return false; 424 } 425 426 // Common code for getting return reg+type, and removing selected instr 427 // from parent occurs here. Instr-specific selection happens in spvSelect(). 428 bool HasDefs = I.getNumDefs() > 0; 429 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 430 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 431 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 432 if (spvSelect(ResVReg, ResType, I)) { 433 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 434 for (unsigned i = 0; i < I.getNumDefs(); ++i) 435 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 436 I.removeFromParent(); 437 return true; 438 } 439 return false; 440 } 441 442 static bool mayApplyGenericSelection(unsigned Opcode) { 443 switch (Opcode) { 444 case TargetOpcode::G_CONSTANT: 445 return false; 446 case TargetOpcode::G_SADDO: 447 case TargetOpcode::G_SSUBO: 448 return true; 449 } 450 return isTypeFoldingSupported(Opcode); 451 } 452 453 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 454 MachineInstr &I) const { 455 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 456 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 457 if (DstRC != SrcRC && SrcRC) 458 MRI->setRegClass(DestReg, SrcRC); 459 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 460 TII.get(TargetOpcode::COPY)) 461 .addDef(DestReg) 462 .addUse(SrcReg) 463 .constrainAllUses(TII, TRI, RBI); 464 } 465 466 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 467 const SPIRVType *ResType, 468 MachineInstr &I) const { 469 const unsigned Opcode = I.getOpcode(); 470 if (mayApplyGenericSelection(Opcode)) 471 return selectImpl(I, *CoverageInfo); 472 switch (Opcode) { 473 case TargetOpcode::G_CONSTANT: 474 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 475 I); 476 case TargetOpcode::G_GLOBAL_VALUE: 477 return selectGlobalValue(ResVReg, I); 478 case TargetOpcode::G_IMPLICIT_DEF: 479 return selectOpUndef(ResVReg, ResType, I); 480 case TargetOpcode::G_FREEZE: 481 return selectFreeze(ResVReg, ResType, I); 482 483 case TargetOpcode::G_INTRINSIC: 484 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 485 case TargetOpcode::G_INTRINSIC_CONVERGENT: 486 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 487 return selectIntrinsic(ResVReg, ResType, I); 488 case TargetOpcode::G_BITREVERSE: 489 return selectBitreverse(ResVReg, ResType, I); 490 491 case TargetOpcode::G_BUILD_VECTOR: 492 return selectBuildVector(ResVReg, ResType, I); 493 case TargetOpcode::G_SPLAT_VECTOR: 494 return selectSplatVector(ResVReg, ResType, I); 495 496 case TargetOpcode::G_SHUFFLE_VECTOR: { 497 MachineBasicBlock &BB = *I.getParent(); 498 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 499 .addDef(ResVReg) 500 .addUse(GR.getSPIRVTypeID(ResType)) 501 .addUse(I.getOperand(1).getReg()) 502 .addUse(I.getOperand(2).getReg()); 503 for (auto V : I.getOperand(3).getShuffleMask()) 504 MIB.addImm(V); 505 return MIB.constrainAllUses(TII, TRI, RBI); 506 } 507 case TargetOpcode::G_MEMMOVE: 508 case TargetOpcode::G_MEMCPY: 509 case TargetOpcode::G_MEMSET: 510 return selectMemOperation(ResVReg, I); 511 512 case TargetOpcode::G_ICMP: 513 return selectICmp(ResVReg, ResType, I); 514 case TargetOpcode::G_FCMP: 515 return selectFCmp(ResVReg, ResType, I); 516 517 case TargetOpcode::G_FRAME_INDEX: 518 return selectFrameIndex(ResVReg, ResType, I); 519 520 case TargetOpcode::G_LOAD: 521 return selectLoad(ResVReg, ResType, I); 522 case TargetOpcode::G_STORE: 523 return selectStore(I); 524 525 case TargetOpcode::G_BR: 526 return selectBranch(I); 527 case TargetOpcode::G_BRCOND: 528 return selectBranchCond(I); 529 530 case TargetOpcode::G_PHI: 531 return selectPhi(ResVReg, ResType, I); 532 533 case TargetOpcode::G_FPTOSI: 534 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 535 case TargetOpcode::G_FPTOUI: 536 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 537 538 case TargetOpcode::G_SITOFP: 539 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 540 case TargetOpcode::G_UITOFP: 541 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 542 543 case TargetOpcode::G_CTPOP: 544 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 545 case TargetOpcode::G_SMIN: 546 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 547 case TargetOpcode::G_UMIN: 548 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 549 550 case TargetOpcode::G_SMAX: 551 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 552 case TargetOpcode::G_UMAX: 553 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 554 555 case TargetOpcode::G_SCMP: 556 return selectSUCmp(ResVReg, ResType, I, true); 557 case TargetOpcode::G_UCMP: 558 return selectSUCmp(ResVReg, ResType, I, false); 559 560 case TargetOpcode::G_FMA: 561 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 562 563 case TargetOpcode::G_FPOW: 564 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 565 case TargetOpcode::G_FPOWI: 566 return selectExtInst(ResVReg, ResType, I, CL::pown); 567 568 case TargetOpcode::G_FEXP: 569 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 570 case TargetOpcode::G_FEXP2: 571 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 572 573 case TargetOpcode::G_FLOG: 574 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 575 case TargetOpcode::G_FLOG2: 576 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 577 case TargetOpcode::G_FLOG10: 578 return selectLog10(ResVReg, ResType, I); 579 580 case TargetOpcode::G_FABS: 581 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 582 case TargetOpcode::G_ABS: 583 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 584 585 case TargetOpcode::G_FMINNUM: 586 case TargetOpcode::G_FMINIMUM: 587 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 588 case TargetOpcode::G_FMAXNUM: 589 case TargetOpcode::G_FMAXIMUM: 590 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 591 592 case TargetOpcode::G_FCOPYSIGN: 593 return selectExtInst(ResVReg, ResType, I, CL::copysign); 594 595 case TargetOpcode::G_FCEIL: 596 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 597 case TargetOpcode::G_FFLOOR: 598 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 599 600 case TargetOpcode::G_FCOS: 601 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 602 case TargetOpcode::G_FSIN: 603 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 604 case TargetOpcode::G_FTAN: 605 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 606 case TargetOpcode::G_FACOS: 607 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 608 case TargetOpcode::G_FASIN: 609 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 610 case TargetOpcode::G_FATAN: 611 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 612 case TargetOpcode::G_FATAN2: 613 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 614 case TargetOpcode::G_FCOSH: 615 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 616 case TargetOpcode::G_FSINH: 617 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 618 case TargetOpcode::G_FTANH: 619 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 620 621 case TargetOpcode::G_FSQRT: 622 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 623 624 case TargetOpcode::G_CTTZ: 625 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 626 return selectExtInst(ResVReg, ResType, I, CL::ctz); 627 case TargetOpcode::G_CTLZ: 628 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 629 return selectExtInst(ResVReg, ResType, I, CL::clz); 630 631 case TargetOpcode::G_INTRINSIC_ROUND: 632 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 633 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 634 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 635 case TargetOpcode::G_INTRINSIC_TRUNC: 636 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 637 case TargetOpcode::G_FRINT: 638 case TargetOpcode::G_FNEARBYINT: 639 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 640 641 case TargetOpcode::G_SMULH: 642 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 643 case TargetOpcode::G_UMULH: 644 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 645 646 case TargetOpcode::G_SADDSAT: 647 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 648 case TargetOpcode::G_UADDSAT: 649 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 650 case TargetOpcode::G_SSUBSAT: 651 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 652 case TargetOpcode::G_USUBSAT: 653 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 654 655 case TargetOpcode::G_UADDO: 656 return selectOverflowArith(ResVReg, ResType, I, 657 ResType->getOpcode() == SPIRV::OpTypeVector 658 ? SPIRV::OpIAddCarryV 659 : SPIRV::OpIAddCarryS); 660 case TargetOpcode::G_USUBO: 661 return selectOverflowArith(ResVReg, ResType, I, 662 ResType->getOpcode() == SPIRV::OpTypeVector 663 ? SPIRV::OpISubBorrowV 664 : SPIRV::OpISubBorrowS); 665 case TargetOpcode::G_UMULO: 666 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 667 case TargetOpcode::G_SMULO: 668 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 669 670 case TargetOpcode::G_SEXT: 671 return selectExt(ResVReg, ResType, I, true); 672 case TargetOpcode::G_ANYEXT: 673 case TargetOpcode::G_ZEXT: 674 return selectExt(ResVReg, ResType, I, false); 675 case TargetOpcode::G_TRUNC: 676 return selectTrunc(ResVReg, ResType, I); 677 case TargetOpcode::G_FPTRUNC: 678 case TargetOpcode::G_FPEXT: 679 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 680 681 case TargetOpcode::G_PTRTOINT: 682 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 683 case TargetOpcode::G_INTTOPTR: 684 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 685 case TargetOpcode::G_BITCAST: 686 return selectBitcast(ResVReg, ResType, I); 687 case TargetOpcode::G_ADDRSPACE_CAST: 688 return selectAddrSpaceCast(ResVReg, ResType, I); 689 case TargetOpcode::G_PTR_ADD: { 690 // Currently, we get G_PTR_ADD only applied to global variables. 691 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 692 Register GV = I.getOperand(1).getReg(); 693 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 694 (void)II; 695 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 696 (*II).getOpcode() == TargetOpcode::COPY || 697 (*II).getOpcode() == SPIRV::OpVariable) && 698 isImm(I.getOperand(2), MRI)); 699 // It may be the initialization of a global variable. 700 bool IsGVInit = false; 701 for (MachineRegisterInfo::use_instr_iterator 702 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 703 UseEnd = MRI->use_instr_end(); 704 UseIt != UseEnd; UseIt = std::next(UseIt)) { 705 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 706 (*UseIt).getOpcode() == SPIRV::OpVariable) { 707 IsGVInit = true; 708 break; 709 } 710 } 711 MachineBasicBlock &BB = *I.getParent(); 712 if (!IsGVInit) { 713 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 714 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 715 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 716 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 717 // Build a new virtual register that is associated with the required 718 // data type. 719 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 720 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 721 // Having a correctly typed base we are ready to build the actually 722 // required GEP. It may not be a constant though, because all Operands 723 // of OpSpecConstantOp is to originate from other const instructions, 724 // and only the AccessChain named opcodes accept a global OpVariable 725 // instruction. We can't use an AccessChain opcode because of the type 726 // mismatch between result and base types. 727 if (!GR.isBitcastCompatible(ResType, GVType)) 728 report_fatal_error( 729 "incompatible result and operand types in a bitcast"); 730 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 731 MachineInstrBuilder MIB = 732 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 733 .addDef(NewVReg) 734 .addUse(ResTypeReg) 735 .addUse(GV); 736 return MIB.constrainAllUses(TII, TRI, RBI) && 737 BuildMI(BB, I, I.getDebugLoc(), 738 TII.get(STI.isVulkanEnv() 739 ? SPIRV::OpInBoundsAccessChain 740 : SPIRV::OpInBoundsPtrAccessChain)) 741 .addDef(ResVReg) 742 .addUse(ResTypeReg) 743 .addUse(NewVReg) 744 .addUse(I.getOperand(2).getReg()) 745 .constrainAllUses(TII, TRI, RBI); 746 } else { 747 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 748 .addDef(ResVReg) 749 .addUse(GR.getSPIRVTypeID(ResType)) 750 .addImm( 751 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 752 .addUse(GV) 753 .addUse(I.getOperand(2).getReg()) 754 .constrainAllUses(TII, TRI, RBI); 755 } 756 } 757 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 758 // initialize a global variable with a constant expression (e.g., the test 759 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 760 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 761 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 762 .addDef(ResVReg) 763 .addUse(GR.getSPIRVTypeID(ResType)) 764 .addImm(static_cast<uint32_t>( 765 SPIRV::Opcode::InBoundsPtrAccessChain)) 766 .addUse(GV) 767 .addUse(Idx) 768 .addUse(I.getOperand(2).getReg()); 769 return MIB.constrainAllUses(TII, TRI, RBI); 770 } 771 772 case TargetOpcode::G_ATOMICRMW_OR: 773 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 774 case TargetOpcode::G_ATOMICRMW_ADD: 775 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 776 case TargetOpcode::G_ATOMICRMW_AND: 777 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 778 case TargetOpcode::G_ATOMICRMW_MAX: 779 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 780 case TargetOpcode::G_ATOMICRMW_MIN: 781 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 782 case TargetOpcode::G_ATOMICRMW_SUB: 783 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 784 case TargetOpcode::G_ATOMICRMW_XOR: 785 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 786 case TargetOpcode::G_ATOMICRMW_UMAX: 787 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 788 case TargetOpcode::G_ATOMICRMW_UMIN: 789 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 790 case TargetOpcode::G_ATOMICRMW_XCHG: 791 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 792 case TargetOpcode::G_ATOMIC_CMPXCHG: 793 return selectAtomicCmpXchg(ResVReg, ResType, I); 794 795 case TargetOpcode::G_ATOMICRMW_FADD: 796 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 797 case TargetOpcode::G_ATOMICRMW_FSUB: 798 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 799 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 800 SPIRV::OpFNegate); 801 case TargetOpcode::G_ATOMICRMW_FMIN: 802 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 803 case TargetOpcode::G_ATOMICRMW_FMAX: 804 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 805 806 case TargetOpcode::G_FENCE: 807 return selectFence(I); 808 809 case TargetOpcode::G_STACKSAVE: 810 return selectStackSave(ResVReg, ResType, I); 811 case TargetOpcode::G_STACKRESTORE: 812 return selectStackRestore(I); 813 814 case TargetOpcode::G_UNMERGE_VALUES: 815 return selectUnmergeValues(I); 816 817 // Discard gen opcodes for intrinsics which we do not expect to actually 818 // represent code after lowering or intrinsics which are not implemented but 819 // should not crash when found in a customer's LLVM IR input. 820 case TargetOpcode::G_TRAP: 821 case TargetOpcode::G_DEBUGTRAP: 822 case TargetOpcode::G_UBSANTRAP: 823 case TargetOpcode::DBG_LABEL: 824 return true; 825 826 default: 827 return false; 828 } 829 } 830 831 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 832 const SPIRVType *ResType, 833 MachineInstr &I, 834 GL::GLSLExtInst GLInst) const { 835 return selectExtInst(ResVReg, ResType, I, 836 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 837 } 838 839 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 840 const SPIRVType *ResType, 841 MachineInstr &I, 842 CL::OpenCLExtInst CLInst) const { 843 return selectExtInst(ResVReg, ResType, I, 844 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 845 } 846 847 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 848 const SPIRVType *ResType, 849 MachineInstr &I, 850 CL::OpenCLExtInst CLInst, 851 GL::GLSLExtInst GLInst) const { 852 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 853 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 854 return selectExtInst(ResVReg, ResType, I, ExtInsts); 855 } 856 857 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 858 const SPIRVType *ResType, 859 MachineInstr &I, 860 const ExtInstList &Insts) const { 861 862 for (const auto &Ex : Insts) { 863 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 864 uint32_t Opcode = Ex.second; 865 if (STI.canUseExtInstSet(Set)) { 866 MachineBasicBlock &BB = *I.getParent(); 867 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 868 .addDef(ResVReg) 869 .addUse(GR.getSPIRVTypeID(ResType)) 870 .addImm(static_cast<uint32_t>(Set)) 871 .addImm(Opcode); 872 const unsigned NumOps = I.getNumOperands(); 873 unsigned Index = 1; 874 if (Index < NumOps && 875 I.getOperand(Index).getType() == 876 MachineOperand::MachineOperandType::MO_IntrinsicID) 877 Index = 2; 878 for (; Index < NumOps; ++Index) 879 MIB.add(I.getOperand(Index)); 880 return MIB.constrainAllUses(TII, TRI, RBI); 881 } 882 } 883 return false; 884 } 885 886 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 887 const SPIRVType *ResType, 888 MachineInstr &I, 889 std::vector<Register> Srcs, 890 unsigned Opcode) const { 891 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 892 .addDef(ResVReg) 893 .addUse(GR.getSPIRVTypeID(ResType)); 894 for (Register SReg : Srcs) { 895 MIB.addUse(SReg); 896 } 897 return MIB.constrainAllUses(TII, TRI, RBI); 898 } 899 900 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 901 const SPIRVType *ResType, 902 MachineInstr &I, 903 unsigned Opcode) const { 904 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 905 Register SrcReg = I.getOperand(1).getReg(); 906 bool IsGV = false; 907 for (MachineRegisterInfo::def_instr_iterator DefIt = 908 MRI->def_instr_begin(SrcReg); 909 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 910 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 911 IsGV = true; 912 break; 913 } 914 } 915 if (IsGV) { 916 uint32_t SpecOpcode = 0; 917 switch (Opcode) { 918 case SPIRV::OpConvertPtrToU: 919 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 920 break; 921 case SPIRV::OpConvertUToPtr: 922 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 923 break; 924 } 925 if (SpecOpcode) 926 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 927 TII.get(SPIRV::OpSpecConstantOp)) 928 .addDef(ResVReg) 929 .addUse(GR.getSPIRVTypeID(ResType)) 930 .addImm(SpecOpcode) 931 .addUse(SrcReg) 932 .constrainAllUses(TII, TRI, RBI); 933 } 934 } 935 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 936 Opcode); 937 } 938 939 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 940 const SPIRVType *ResType, 941 MachineInstr &I) const { 942 Register OpReg = I.getOperand(1).getReg(); 943 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 944 if (!GR.isBitcastCompatible(ResType, OpType)) 945 report_fatal_error("incompatible result and operand types in a bitcast"); 946 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 947 } 948 949 static void addMemoryOperands(MachineMemOperand *MemOp, 950 MachineInstrBuilder &MIB) { 951 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 952 if (MemOp->isVolatile()) 953 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 954 if (MemOp->isNonTemporal()) 955 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 956 if (MemOp->getAlign().value()) 957 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 958 959 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 960 MIB.addImm(SpvMemOp); 961 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 962 MIB.addImm(MemOp->getAlign().value()); 963 } 964 } 965 966 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 967 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 968 if (Flags & MachineMemOperand::Flags::MOVolatile) 969 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 970 if (Flags & MachineMemOperand::Flags::MONonTemporal) 971 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 972 973 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 974 MIB.addImm(SpvMemOp); 975 } 976 977 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 978 const SPIRVType *ResType, 979 MachineInstr &I) const { 980 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 981 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 982 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 983 .addDef(ResVReg) 984 .addUse(GR.getSPIRVTypeID(ResType)) 985 .addUse(Ptr); 986 if (!I.getNumMemOperands()) { 987 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 988 I.getOpcode() == 989 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 990 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 991 } else { 992 addMemoryOperands(*I.memoperands_begin(), MIB); 993 } 994 return MIB.constrainAllUses(TII, TRI, RBI); 995 } 996 997 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 998 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 999 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1000 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1001 MachineBasicBlock &BB = *I.getParent(); 1002 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1003 .addUse(Ptr) 1004 .addUse(StoreVal); 1005 if (!I.getNumMemOperands()) { 1006 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1007 I.getOpcode() == 1008 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1009 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1010 } else { 1011 addMemoryOperands(*I.memoperands_begin(), MIB); 1012 } 1013 return MIB.constrainAllUses(TII, TRI, RBI); 1014 } 1015 1016 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1017 const SPIRVType *ResType, 1018 MachineInstr &I) const { 1019 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1020 report_fatal_error( 1021 "llvm.stacksave intrinsic: this instruction requires the following " 1022 "SPIR-V extension: SPV_INTEL_variable_length_array", 1023 false); 1024 MachineBasicBlock &BB = *I.getParent(); 1025 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1026 .addDef(ResVReg) 1027 .addUse(GR.getSPIRVTypeID(ResType)) 1028 .constrainAllUses(TII, TRI, RBI); 1029 } 1030 1031 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1032 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1033 report_fatal_error( 1034 "llvm.stackrestore intrinsic: this instruction requires the following " 1035 "SPIR-V extension: SPV_INTEL_variable_length_array", 1036 false); 1037 if (!I.getOperand(0).isReg()) 1038 return false; 1039 MachineBasicBlock &BB = *I.getParent(); 1040 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1041 .addUse(I.getOperand(0).getReg()) 1042 .constrainAllUses(TII, TRI, RBI); 1043 } 1044 1045 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1046 MachineInstr &I) const { 1047 MachineBasicBlock &BB = *I.getParent(); 1048 Register SrcReg = I.getOperand(1).getReg(); 1049 bool Result = true; 1050 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1051 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1052 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1053 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1054 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1055 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1056 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1057 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1058 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1059 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1060 Function &CurFunction = GR.CurMF->getFunction(); 1061 Type *LLVMArrTy = 1062 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1063 // Module takes ownership of the global var. 1064 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1065 true, GlobalValue::InternalLinkage, 1066 Constant::getNullValue(LLVMArrTy)); 1067 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1068 GR.add(GV, GR.CurMF, VarReg); 1069 1070 Result &= 1071 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1072 .addDef(VarReg) 1073 .addUse(GR.getSPIRVTypeID(VarTy)) 1074 .addImm(SPIRV::StorageClass::UniformConstant) 1075 .addUse(Const) 1076 .constrainAllUses(TII, TRI, RBI); 1077 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1078 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1079 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1080 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1081 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1082 } 1083 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1084 .addUse(I.getOperand(0).getReg()) 1085 .addUse(SrcReg) 1086 .addUse(I.getOperand(2).getReg()); 1087 if (I.getNumMemOperands()) 1088 addMemoryOperands(*I.memoperands_begin(), MIB); 1089 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1090 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1091 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1092 return Result; 1093 } 1094 1095 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1096 const SPIRVType *ResType, 1097 MachineInstr &I, 1098 unsigned NewOpcode, 1099 unsigned NegateOpcode) const { 1100 bool Result = true; 1101 assert(I.hasOneMemOperand()); 1102 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1103 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1104 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1105 auto ScopeConstant = buildI32Constant(Scope, I); 1106 Register ScopeReg = ScopeConstant.first; 1107 Result &= ScopeConstant.second; 1108 1109 Register Ptr = I.getOperand(1).getReg(); 1110 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1111 // auto ScSem = 1112 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1113 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1114 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1115 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1116 Register MemSemReg = MemSemConstant.first; 1117 Result &= MemSemConstant.second; 1118 1119 Register ValueReg = I.getOperand(2).getReg(); 1120 if (NegateOpcode != 0) { 1121 // Translation with negative value operand is requested 1122 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1123 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1124 ValueReg = TmpReg; 1125 } 1126 1127 return Result && 1128 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1129 .addDef(ResVReg) 1130 .addUse(GR.getSPIRVTypeID(ResType)) 1131 .addUse(Ptr) 1132 .addUse(ScopeReg) 1133 .addUse(MemSemReg) 1134 .addUse(ValueReg) 1135 .constrainAllUses(TII, TRI, RBI); 1136 } 1137 1138 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1139 unsigned ArgI = I.getNumOperands() - 1; 1140 Register SrcReg = 1141 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1142 SPIRVType *DefType = 1143 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1144 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1145 report_fatal_error( 1146 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1147 1148 SPIRVType *ScalarType = 1149 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1150 MachineBasicBlock &BB = *I.getParent(); 1151 bool Res = false; 1152 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1153 Register ResVReg = I.getOperand(i).getReg(); 1154 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1155 if (!ResType) { 1156 // There was no "assign type" actions, let's fix this now 1157 ResType = ScalarType; 1158 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1159 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1160 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1161 } 1162 auto MIB = 1163 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1164 .addDef(ResVReg) 1165 .addUse(GR.getSPIRVTypeID(ResType)) 1166 .addUse(SrcReg) 1167 .addImm(static_cast<int64_t>(i)); 1168 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1169 } 1170 return Res; 1171 } 1172 1173 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1174 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1175 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1176 auto MemSemConstant = buildI32Constant(MemSem, I); 1177 Register MemSemReg = MemSemConstant.first; 1178 bool Result = MemSemConstant.second; 1179 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1180 uint32_t Scope = static_cast<uint32_t>( 1181 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1182 auto ScopeConstant = buildI32Constant(Scope, I); 1183 Register ScopeReg = ScopeConstant.first; 1184 Result &= ScopeConstant.second; 1185 MachineBasicBlock &BB = *I.getParent(); 1186 return Result && 1187 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1188 .addUse(ScopeReg) 1189 .addUse(MemSemReg) 1190 .constrainAllUses(TII, TRI, RBI); 1191 } 1192 1193 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1194 const SPIRVType *ResType, 1195 MachineInstr &I, 1196 unsigned Opcode) const { 1197 Type *ResTy = nullptr; 1198 StringRef ResName; 1199 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1200 report_fatal_error( 1201 "Not enough info to select the arithmetic with overflow instruction"); 1202 if (!ResTy || !ResTy->isStructTy()) 1203 report_fatal_error("Expect struct type result for the arithmetic " 1204 "with overflow instruction"); 1205 // "Result Type must be from OpTypeStruct. The struct must have two members, 1206 // and the two members must be the same type." 1207 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1208 ResTy = StructType::get(ResElemTy, ResElemTy); 1209 // Build SPIR-V types and constant(s) if needed. 1210 MachineIRBuilder MIRBuilder(I); 1211 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1212 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1213 assert(I.getNumDefs() > 1 && "Not enought operands"); 1214 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1215 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1216 if (N > 1) 1217 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1218 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1219 Register ZeroReg = buildZerosVal(ResType, I); 1220 // A new virtual register to store the result struct. 1221 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1222 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1223 // Build the result name if needed. 1224 if (ResName.size() > 0) 1225 buildOpName(StructVReg, ResName, MIRBuilder); 1226 // Build the arithmetic with overflow instruction. 1227 MachineBasicBlock &BB = *I.getParent(); 1228 auto MIB = 1229 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1230 .addDef(StructVReg) 1231 .addUse(GR.getSPIRVTypeID(StructType)); 1232 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1233 MIB.addUse(I.getOperand(i).getReg()); 1234 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1235 // Build instructions to extract fields of the instruction's result. 1236 // A new virtual register to store the higher part of the result struct. 1237 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1238 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1239 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1240 auto MIB = 1241 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1242 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1243 .addUse(GR.getSPIRVTypeID(ResType)) 1244 .addUse(StructVReg) 1245 .addImm(i); 1246 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1247 } 1248 // Build boolean value from the higher part. 1249 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1250 .addDef(I.getOperand(1).getReg()) 1251 .addUse(BoolTypeReg) 1252 .addUse(HigherVReg) 1253 .addUse(ZeroReg) 1254 .constrainAllUses(TII, TRI, RBI); 1255 } 1256 1257 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1258 const SPIRVType *ResType, 1259 MachineInstr &I) const { 1260 bool Result = true; 1261 Register ScopeReg; 1262 Register MemSemEqReg; 1263 Register MemSemNeqReg; 1264 Register Ptr = I.getOperand(2).getReg(); 1265 if (!isa<GIntrinsic>(I)) { 1266 assert(I.hasOneMemOperand()); 1267 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1268 unsigned Scope = static_cast<uint32_t>(getMemScope( 1269 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1270 auto ScopeConstant = buildI32Constant(Scope, I); 1271 ScopeReg = ScopeConstant.first; 1272 Result &= ScopeConstant.second; 1273 1274 unsigned ScSem = static_cast<uint32_t>( 1275 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1276 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1277 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1278 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1279 MemSemEqReg = MemSemEqConstant.first; 1280 Result &= MemSemEqConstant.second; 1281 AtomicOrdering FO = MemOp->getFailureOrdering(); 1282 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1283 if (MemSemEq == MemSemNeq) 1284 MemSemNeqReg = MemSemEqReg; 1285 else { 1286 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1287 MemSemNeqReg = MemSemNeqConstant.first; 1288 Result &= MemSemNeqConstant.second; 1289 } 1290 } else { 1291 ScopeReg = I.getOperand(5).getReg(); 1292 MemSemEqReg = I.getOperand(6).getReg(); 1293 MemSemNeqReg = I.getOperand(7).getReg(); 1294 } 1295 1296 Register Cmp = I.getOperand(3).getReg(); 1297 Register Val = I.getOperand(4).getReg(); 1298 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1299 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1300 const DebugLoc &DL = I.getDebugLoc(); 1301 Result &= 1302 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1303 .addDef(ACmpRes) 1304 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1305 .addUse(Ptr) 1306 .addUse(ScopeReg) 1307 .addUse(MemSemEqReg) 1308 .addUse(MemSemNeqReg) 1309 .addUse(Val) 1310 .addUse(Cmp) 1311 .constrainAllUses(TII, TRI, RBI); 1312 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1313 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1314 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1315 .addDef(CmpSuccReg) 1316 .addUse(GR.getSPIRVTypeID(BoolTy)) 1317 .addUse(ACmpRes) 1318 .addUse(Cmp) 1319 .constrainAllUses(TII, TRI, RBI); 1320 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1321 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1322 .addDef(TmpReg) 1323 .addUse(GR.getSPIRVTypeID(ResType)) 1324 .addUse(ACmpRes) 1325 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1326 .addImm(0) 1327 .constrainAllUses(TII, TRI, RBI); 1328 return Result && 1329 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1330 .addDef(ResVReg) 1331 .addUse(GR.getSPIRVTypeID(ResType)) 1332 .addUse(CmpSuccReg) 1333 .addUse(TmpReg) 1334 .addImm(1) 1335 .constrainAllUses(TII, TRI, RBI); 1336 } 1337 1338 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1339 switch (SC) { 1340 case SPIRV::StorageClass::Workgroup: 1341 case SPIRV::StorageClass::CrossWorkgroup: 1342 case SPIRV::StorageClass::Function: 1343 return true; 1344 default: 1345 return false; 1346 } 1347 } 1348 1349 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1350 switch (SC) { 1351 case SPIRV::StorageClass::DeviceOnlyINTEL: 1352 case SPIRV::StorageClass::HostOnlyINTEL: 1353 return true; 1354 default: 1355 return false; 1356 } 1357 } 1358 1359 // Returns true ResVReg is referred only from global vars and OpName's. 1360 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1361 bool IsGRef = false; 1362 bool IsAllowedRefs = 1363 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1364 [&IsGRef](auto const &It) { 1365 unsigned Opcode = It.getOpcode(); 1366 if (Opcode == SPIRV::OpConstantComposite || 1367 Opcode == SPIRV::OpVariable || 1368 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1369 return IsGRef = true; 1370 return Opcode == SPIRV::OpName; 1371 }); 1372 return IsAllowedRefs && IsGRef; 1373 } 1374 1375 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1376 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1377 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1378 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1379 } 1380 1381 MachineInstrBuilder 1382 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1383 Register Src, Register DestType, 1384 uint32_t Opcode) const { 1385 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1386 TII.get(SPIRV::OpSpecConstantOp)) 1387 .addDef(Dest) 1388 .addUse(DestType) 1389 .addImm(Opcode) 1390 .addUse(Src); 1391 } 1392 1393 MachineInstrBuilder 1394 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1395 SPIRVType *SrcPtrTy) const { 1396 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1397 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1398 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1399 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1400 SPIRV::StorageClass::Generic), 1401 GR.getPointerSize())); 1402 MachineFunction *MF = I.getParent()->getParent(); 1403 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1404 MachineInstrBuilder MIB = buildSpecConstantOp( 1405 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1406 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1407 GR.add(MIB.getInstr(), MF, Tmp); 1408 return MIB; 1409 } 1410 1411 // In SPIR-V address space casting can only happen to and from the Generic 1412 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1413 // pointers to and from Generic pointers. As such, we can convert e.g. from 1414 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1415 // other combinations can only be done by a bitcast, and are probably not safe. 1416 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1417 const SPIRVType *ResType, 1418 MachineInstr &I) const { 1419 MachineBasicBlock &BB = *I.getParent(); 1420 const DebugLoc &DL = I.getDebugLoc(); 1421 1422 Register SrcPtr = I.getOperand(1).getReg(); 1423 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1424 1425 // don't generate a cast for a null that may be represented by OpTypeInt 1426 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1427 ResType->getOpcode() != SPIRV::OpTypePointer) 1428 return BuildCOPY(ResVReg, SrcPtr, I); 1429 1430 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1431 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1432 1433 if (isASCastInGVar(MRI, ResVReg)) { 1434 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1435 // are expressed by OpSpecConstantOp with an Opcode. 1436 // TODO: maybe insert a check whether the Kernel capability was declared and 1437 // so PtrCastToGeneric/GenericCastToPtr are available. 1438 unsigned SpecOpcode = 1439 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1440 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1441 : (SrcSC == SPIRV::StorageClass::Generic && 1442 isGenericCastablePtr(DstSC) 1443 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1444 : 0); 1445 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1446 // correct value of ResType and use general i8* instead. Maybe this should 1447 // be addressed in the emit-intrinsic step to infer a correct 1448 // OpConstantComposite type. 1449 if (SpecOpcode) { 1450 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1451 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1452 .constrainAllUses(TII, TRI, RBI); 1453 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1454 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1455 return MIB.constrainAllUses(TII, TRI, RBI) && 1456 buildSpecConstantOp( 1457 I, ResVReg, MIB->getOperand(0).getReg(), 1458 getUcharPtrTypeReg(I, DstSC), 1459 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1460 .constrainAllUses(TII, TRI, RBI); 1461 } 1462 } 1463 1464 // don't generate a cast between identical storage classes 1465 if (SrcSC == DstSC) 1466 return BuildCOPY(ResVReg, SrcPtr, I); 1467 1468 if ((SrcSC == SPIRV::StorageClass::Function && 1469 DstSC == SPIRV::StorageClass::Private) || 1470 (DstSC == SPIRV::StorageClass::Function && 1471 SrcSC == SPIRV::StorageClass::Private)) 1472 return BuildCOPY(ResVReg, SrcPtr, I); 1473 1474 // Casting from an eligible pointer to Generic. 1475 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1476 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1477 // Casting from Generic to an eligible pointer. 1478 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1479 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1480 // Casting between 2 eligible pointers using Generic as an intermediary. 1481 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1482 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1483 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1484 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1485 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1486 .addDef(Tmp) 1487 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1488 .addUse(SrcPtr) 1489 .constrainAllUses(TII, TRI, RBI); 1490 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1491 .addDef(ResVReg) 1492 .addUse(GR.getSPIRVTypeID(ResType)) 1493 .addUse(Tmp) 1494 .constrainAllUses(TII, TRI, RBI); 1495 } 1496 1497 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1498 // be applied 1499 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1500 return selectUnOp(ResVReg, ResType, I, 1501 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1502 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1503 return selectUnOp(ResVReg, ResType, I, 1504 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1505 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1506 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1507 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1508 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1509 1510 // Bitcast for pointers requires that the address spaces must match 1511 return false; 1512 } 1513 1514 static unsigned getFCmpOpcode(unsigned PredNum) { 1515 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1516 switch (Pred) { 1517 case CmpInst::FCMP_OEQ: 1518 return SPIRV::OpFOrdEqual; 1519 case CmpInst::FCMP_OGE: 1520 return SPIRV::OpFOrdGreaterThanEqual; 1521 case CmpInst::FCMP_OGT: 1522 return SPIRV::OpFOrdGreaterThan; 1523 case CmpInst::FCMP_OLE: 1524 return SPIRV::OpFOrdLessThanEqual; 1525 case CmpInst::FCMP_OLT: 1526 return SPIRV::OpFOrdLessThan; 1527 case CmpInst::FCMP_ONE: 1528 return SPIRV::OpFOrdNotEqual; 1529 case CmpInst::FCMP_ORD: 1530 return SPIRV::OpOrdered; 1531 case CmpInst::FCMP_UEQ: 1532 return SPIRV::OpFUnordEqual; 1533 case CmpInst::FCMP_UGE: 1534 return SPIRV::OpFUnordGreaterThanEqual; 1535 case CmpInst::FCMP_UGT: 1536 return SPIRV::OpFUnordGreaterThan; 1537 case CmpInst::FCMP_ULE: 1538 return SPIRV::OpFUnordLessThanEqual; 1539 case CmpInst::FCMP_ULT: 1540 return SPIRV::OpFUnordLessThan; 1541 case CmpInst::FCMP_UNE: 1542 return SPIRV::OpFUnordNotEqual; 1543 case CmpInst::FCMP_UNO: 1544 return SPIRV::OpUnordered; 1545 default: 1546 llvm_unreachable("Unknown predicate type for FCmp"); 1547 } 1548 } 1549 1550 static unsigned getICmpOpcode(unsigned PredNum) { 1551 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1552 switch (Pred) { 1553 case CmpInst::ICMP_EQ: 1554 return SPIRV::OpIEqual; 1555 case CmpInst::ICMP_NE: 1556 return SPIRV::OpINotEqual; 1557 case CmpInst::ICMP_SGE: 1558 return SPIRV::OpSGreaterThanEqual; 1559 case CmpInst::ICMP_SGT: 1560 return SPIRV::OpSGreaterThan; 1561 case CmpInst::ICMP_SLE: 1562 return SPIRV::OpSLessThanEqual; 1563 case CmpInst::ICMP_SLT: 1564 return SPIRV::OpSLessThan; 1565 case CmpInst::ICMP_UGE: 1566 return SPIRV::OpUGreaterThanEqual; 1567 case CmpInst::ICMP_UGT: 1568 return SPIRV::OpUGreaterThan; 1569 case CmpInst::ICMP_ULE: 1570 return SPIRV::OpULessThanEqual; 1571 case CmpInst::ICMP_ULT: 1572 return SPIRV::OpULessThan; 1573 default: 1574 llvm_unreachable("Unknown predicate type for ICmp"); 1575 } 1576 } 1577 1578 static unsigned getPtrCmpOpcode(unsigned Pred) { 1579 switch (static_cast<CmpInst::Predicate>(Pred)) { 1580 case CmpInst::ICMP_EQ: 1581 return SPIRV::OpPtrEqual; 1582 case CmpInst::ICMP_NE: 1583 return SPIRV::OpPtrNotEqual; 1584 default: 1585 llvm_unreachable("Unknown predicate type for pointer comparison"); 1586 } 1587 } 1588 1589 // Return the logical operation, or abort if none exists. 1590 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1591 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1592 switch (Pred) { 1593 case CmpInst::ICMP_EQ: 1594 return SPIRV::OpLogicalEqual; 1595 case CmpInst::ICMP_NE: 1596 return SPIRV::OpLogicalNotEqual; 1597 default: 1598 llvm_unreachable("Unknown predicate type for Bool comparison"); 1599 } 1600 } 1601 1602 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1603 if (!LLVMFloatTy) 1604 return APFloat::getZero(APFloat::IEEEsingle()); 1605 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1606 case Type::HalfTyID: 1607 return APFloat::getZero(APFloat::IEEEhalf()); 1608 default: 1609 case Type::FloatTyID: 1610 return APFloat::getZero(APFloat::IEEEsingle()); 1611 case Type::DoubleTyID: 1612 return APFloat::getZero(APFloat::IEEEdouble()); 1613 } 1614 } 1615 1616 static APFloat getOneFP(const Type *LLVMFloatTy) { 1617 if (!LLVMFloatTy) 1618 return APFloat::getOne(APFloat::IEEEsingle()); 1619 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1620 case Type::HalfTyID: 1621 return APFloat::getOne(APFloat::IEEEhalf()); 1622 default: 1623 case Type::FloatTyID: 1624 return APFloat::getOne(APFloat::IEEEsingle()); 1625 case Type::DoubleTyID: 1626 return APFloat::getOne(APFloat::IEEEdouble()); 1627 } 1628 } 1629 1630 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1631 const SPIRVType *ResType, 1632 MachineInstr &I, 1633 unsigned OpAnyOrAll) const { 1634 assert(I.getNumOperands() == 3); 1635 assert(I.getOperand(2).isReg()); 1636 MachineBasicBlock &BB = *I.getParent(); 1637 Register InputRegister = I.getOperand(2).getReg(); 1638 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1639 1640 if (!InputType) 1641 report_fatal_error("Input Type could not be determined."); 1642 1643 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1644 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1645 if (IsBoolTy && !IsVectorTy) { 1646 assert(ResVReg == I.getOperand(0).getReg()); 1647 return BuildCOPY(ResVReg, InputRegister, I); 1648 } 1649 1650 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1651 unsigned SpirvNotEqualId = 1652 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1653 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1654 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1655 Register NotEqualReg = ResVReg; 1656 1657 if (IsVectorTy) { 1658 NotEqualReg = IsBoolTy ? InputRegister 1659 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1660 const unsigned NumElts = InputType->getOperand(2).getImm(); 1661 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1662 } 1663 1664 bool Result = true; 1665 if (!IsBoolTy) { 1666 Register ConstZeroReg = 1667 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1668 1669 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1670 .addDef(NotEqualReg) 1671 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1672 .addUse(InputRegister) 1673 .addUse(ConstZeroReg) 1674 .constrainAllUses(TII, TRI, RBI); 1675 } 1676 1677 if (!IsVectorTy) 1678 return Result; 1679 1680 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1681 .addDef(ResVReg) 1682 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1683 .addUse(NotEqualReg) 1684 .constrainAllUses(TII, TRI, RBI); 1685 } 1686 1687 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1688 const SPIRVType *ResType, 1689 MachineInstr &I) const { 1690 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1691 } 1692 1693 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1694 const SPIRVType *ResType, 1695 MachineInstr &I) const { 1696 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1697 } 1698 1699 // Select the OpDot instruction for the given float dot 1700 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1701 const SPIRVType *ResType, 1702 MachineInstr &I) const { 1703 assert(I.getNumOperands() == 4); 1704 assert(I.getOperand(2).isReg()); 1705 assert(I.getOperand(3).isReg()); 1706 1707 [[maybe_unused]] SPIRVType *VecType = 1708 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1709 1710 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1711 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1712 "dot product requires a vector of at least 2 components"); 1713 1714 [[maybe_unused]] SPIRVType *EltType = 1715 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1716 1717 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1718 1719 MachineBasicBlock &BB = *I.getParent(); 1720 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1721 .addDef(ResVReg) 1722 .addUse(GR.getSPIRVTypeID(ResType)) 1723 .addUse(I.getOperand(2).getReg()) 1724 .addUse(I.getOperand(3).getReg()) 1725 .constrainAllUses(TII, TRI, RBI); 1726 } 1727 1728 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1729 const SPIRVType *ResType, 1730 MachineInstr &I, 1731 bool Signed) const { 1732 assert(I.getNumOperands() == 4); 1733 assert(I.getOperand(2).isReg()); 1734 assert(I.getOperand(3).isReg()); 1735 MachineBasicBlock &BB = *I.getParent(); 1736 1737 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1738 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1739 .addDef(ResVReg) 1740 .addUse(GR.getSPIRVTypeID(ResType)) 1741 .addUse(I.getOperand(2).getReg()) 1742 .addUse(I.getOperand(3).getReg()) 1743 .constrainAllUses(TII, TRI, RBI); 1744 } 1745 1746 // Since pre-1.6 SPIRV has no integer dot implementation, 1747 // expand by piecewise multiplying and adding the results 1748 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1749 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1750 assert(I.getNumOperands() == 4); 1751 assert(I.getOperand(2).isReg()); 1752 assert(I.getOperand(3).isReg()); 1753 MachineBasicBlock &BB = *I.getParent(); 1754 1755 // Multiply the vectors, then sum the results 1756 Register Vec0 = I.getOperand(2).getReg(); 1757 Register Vec1 = I.getOperand(3).getReg(); 1758 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1759 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1760 1761 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1762 .addDef(TmpVec) 1763 .addUse(GR.getSPIRVTypeID(VecType)) 1764 .addUse(Vec0) 1765 .addUse(Vec1) 1766 .constrainAllUses(TII, TRI, RBI); 1767 1768 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1769 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1770 "dot product requires a vector of at least 2 components"); 1771 1772 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1773 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1774 .addDef(Res) 1775 .addUse(GR.getSPIRVTypeID(ResType)) 1776 .addUse(TmpVec) 1777 .addImm(0) 1778 .constrainAllUses(TII, TRI, RBI); 1779 1780 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1781 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1782 1783 Result &= 1784 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1785 .addDef(Elt) 1786 .addUse(GR.getSPIRVTypeID(ResType)) 1787 .addUse(TmpVec) 1788 .addImm(i) 1789 .constrainAllUses(TII, TRI, RBI); 1790 1791 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1792 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1793 : ResVReg; 1794 1795 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1796 .addDef(Sum) 1797 .addUse(GR.getSPIRVTypeID(ResType)) 1798 .addUse(Res) 1799 .addUse(Elt) 1800 .constrainAllUses(TII, TRI, RBI); 1801 Res = Sum; 1802 } 1803 1804 return Result; 1805 } 1806 1807 template <bool Signed> 1808 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1809 const SPIRVType *ResType, 1810 MachineInstr &I) const { 1811 assert(I.getNumOperands() == 5); 1812 assert(I.getOperand(2).isReg()); 1813 assert(I.getOperand(3).isReg()); 1814 assert(I.getOperand(4).isReg()); 1815 MachineBasicBlock &BB = *I.getParent(); 1816 1817 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1818 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1819 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1820 .addDef(Dot) 1821 .addUse(GR.getSPIRVTypeID(ResType)) 1822 .addUse(I.getOperand(2).getReg()) 1823 .addUse(I.getOperand(3).getReg()) 1824 .constrainAllUses(TII, TRI, RBI); 1825 1826 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1827 .addDef(ResVReg) 1828 .addUse(GR.getSPIRVTypeID(ResType)) 1829 .addUse(Dot) 1830 .addUse(I.getOperand(4).getReg()) 1831 .constrainAllUses(TII, TRI, RBI); 1832 } 1833 1834 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1835 // extract the elements of the packed inputs, multiply them and add the result 1836 // to the accumulator. 1837 template <bool Signed> 1838 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1839 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1840 assert(I.getNumOperands() == 5); 1841 assert(I.getOperand(2).isReg()); 1842 assert(I.getOperand(3).isReg()); 1843 assert(I.getOperand(4).isReg()); 1844 MachineBasicBlock &BB = *I.getParent(); 1845 1846 bool Result = true; 1847 1848 // Acc = C 1849 Register Acc = I.getOperand(4).getReg(); 1850 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1851 auto ExtractOp = 1852 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1853 1854 // Extract the i8 element, multiply and add it to the accumulator 1855 for (unsigned i = 0; i < 4; i++) { 1856 // A[i] 1857 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1858 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1859 .addDef(AElt) 1860 .addUse(GR.getSPIRVTypeID(ResType)) 1861 .addUse(I.getOperand(2).getReg()) 1862 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1863 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1864 .constrainAllUses(TII, TRI, RBI); 1865 1866 // B[i] 1867 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1868 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1869 .addDef(BElt) 1870 .addUse(GR.getSPIRVTypeID(ResType)) 1871 .addUse(I.getOperand(3).getReg()) 1872 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1873 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1874 .constrainAllUses(TII, TRI, RBI); 1875 1876 // A[i] * B[i] 1877 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1878 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1879 .addDef(Mul) 1880 .addUse(GR.getSPIRVTypeID(ResType)) 1881 .addUse(AElt) 1882 .addUse(BElt) 1883 .constrainAllUses(TII, TRI, RBI); 1884 1885 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1886 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1887 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1888 .addDef(MaskMul) 1889 .addUse(GR.getSPIRVTypeID(ResType)) 1890 .addUse(Mul) 1891 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1892 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1893 .constrainAllUses(TII, TRI, RBI); 1894 1895 // Acc = Acc + A[i] * B[i] 1896 Register Sum = 1897 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1898 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1899 .addDef(Sum) 1900 .addUse(GR.getSPIRVTypeID(ResType)) 1901 .addUse(Acc) 1902 .addUse(MaskMul) 1903 .constrainAllUses(TII, TRI, RBI); 1904 1905 Acc = Sum; 1906 } 1907 1908 return Result; 1909 } 1910 1911 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1912 /// does not have a saturate builtin. 1913 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1914 const SPIRVType *ResType, 1915 MachineInstr &I) const { 1916 assert(I.getNumOperands() == 3); 1917 assert(I.getOperand(2).isReg()); 1918 MachineBasicBlock &BB = *I.getParent(); 1919 Register VZero = buildZerosValF(ResType, I); 1920 Register VOne = buildOnesValF(ResType, I); 1921 1922 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1923 .addDef(ResVReg) 1924 .addUse(GR.getSPIRVTypeID(ResType)) 1925 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1926 .addImm(GL::FClamp) 1927 .addUse(I.getOperand(2).getReg()) 1928 .addUse(VZero) 1929 .addUse(VOne) 1930 .constrainAllUses(TII, TRI, RBI); 1931 } 1932 1933 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1934 const SPIRVType *ResType, 1935 MachineInstr &I) const { 1936 assert(I.getNumOperands() == 3); 1937 assert(I.getOperand(2).isReg()); 1938 MachineBasicBlock &BB = *I.getParent(); 1939 Register InputRegister = I.getOperand(2).getReg(); 1940 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1941 auto &DL = I.getDebugLoc(); 1942 1943 if (!InputType) 1944 report_fatal_error("Input Type could not be determined."); 1945 1946 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1947 1948 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1949 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1950 1951 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1952 1953 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1954 Register SignReg = NeedsConversion 1955 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1956 : ResVReg; 1957 1958 bool Result = 1959 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1960 .addDef(SignReg) 1961 .addUse(GR.getSPIRVTypeID(InputType)) 1962 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1963 .addImm(SignOpcode) 1964 .addUse(InputRegister) 1965 .constrainAllUses(TII, TRI, RBI); 1966 1967 if (NeedsConversion) { 1968 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1969 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1970 .addDef(ResVReg) 1971 .addUse(GR.getSPIRVTypeID(ResType)) 1972 .addUse(SignReg) 1973 .constrainAllUses(TII, TRI, RBI); 1974 } 1975 1976 return Result; 1977 } 1978 1979 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 1980 const SPIRVType *ResType, 1981 MachineInstr &I, 1982 unsigned Opcode) const { 1983 MachineBasicBlock &BB = *I.getParent(); 1984 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1985 1986 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1987 .addDef(ResVReg) 1988 .addUse(GR.getSPIRVTypeID(ResType)) 1989 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 1990 IntTy, TII)); 1991 1992 for (unsigned J = 2; J < I.getNumOperands(); J++) { 1993 BMI.addUse(I.getOperand(J).getReg()); 1994 } 1995 1996 return BMI.constrainAllUses(TII, TRI, RBI); 1997 } 1998 1999 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2000 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2001 2002 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2003 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2004 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2005 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2006 SPIRV::OpGroupNonUniformBallot); 2007 2008 MachineBasicBlock &BB = *I.getParent(); 2009 Result &= 2010 BuildMI(BB, I, I.getDebugLoc(), 2011 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2012 .addDef(ResVReg) 2013 .addUse(GR.getSPIRVTypeID(ResType)) 2014 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2015 .addImm(SPIRV::GroupOperation::Reduce) 2016 .addUse(BallotReg) 2017 .constrainAllUses(TII, TRI, RBI); 2018 2019 return Result; 2020 } 2021 2022 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2023 const SPIRVType *ResType, 2024 MachineInstr &I) const { 2025 MachineBasicBlock &BB = *I.getParent(); 2026 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2027 .addDef(ResVReg) 2028 .addUse(GR.getSPIRVTypeID(ResType)) 2029 .addUse(I.getOperand(1).getReg()) 2030 .constrainAllUses(TII, TRI, RBI); 2031 } 2032 2033 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2034 const SPIRVType *ResType, 2035 MachineInstr &I) const { 2036 // There is no way to implement `freeze` correctly without support on SPIR-V 2037 // standard side, but we may at least address a simple (static) case when 2038 // undef/poison value presence is obvious. The main benefit of even 2039 // incomplete `freeze` support is preventing of translation from crashing due 2040 // to lack of support on legalization and instruction selection steps. 2041 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2042 return false; 2043 Register OpReg = I.getOperand(1).getReg(); 2044 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2045 Register Reg; 2046 switch (Def->getOpcode()) { 2047 case SPIRV::ASSIGN_TYPE: 2048 if (MachineInstr *AssignToDef = 2049 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2050 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2051 Reg = Def->getOperand(2).getReg(); 2052 } 2053 break; 2054 case SPIRV::OpUndef: 2055 Reg = Def->getOperand(1).getReg(); 2056 break; 2057 } 2058 unsigned DestOpCode; 2059 if (Reg.isValid()) { 2060 DestOpCode = SPIRV::OpConstantNull; 2061 } else { 2062 DestOpCode = TargetOpcode::COPY; 2063 Reg = OpReg; 2064 } 2065 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2066 .addDef(I.getOperand(0).getReg()) 2067 .addUse(Reg) 2068 .constrainAllUses(TII, TRI, RBI); 2069 } 2070 return false; 2071 } 2072 2073 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2074 const SPIRVType *ResType) { 2075 Register OpReg = ResType->getOperand(2).getReg(); 2076 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2077 if (!OpDef) 2078 return 0; 2079 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2080 OpDef->getOperand(1).isReg()) { 2081 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2082 OpDef = RefDef; 2083 } 2084 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2085 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2086 : 0; 2087 return N; 2088 } 2089 2090 // Return true if the type represents a constant register 2091 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2092 SmallPtrSet<SPIRVType *, 4> &Visited) { 2093 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2094 OpDef->getOperand(1).isReg()) { 2095 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2096 OpDef = RefDef; 2097 } 2098 2099 if (Visited.contains(OpDef)) 2100 return true; 2101 Visited.insert(OpDef); 2102 2103 unsigned Opcode = OpDef->getOpcode(); 2104 switch (Opcode) { 2105 case TargetOpcode::G_CONSTANT: 2106 case TargetOpcode::G_FCONSTANT: 2107 return true; 2108 case TargetOpcode::G_INTRINSIC: 2109 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2110 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2111 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2112 Intrinsic::spv_const_composite; 2113 case TargetOpcode::G_BUILD_VECTOR: 2114 case TargetOpcode::G_SPLAT_VECTOR: { 2115 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2116 i++) { 2117 SPIRVType *OpNestedDef = 2118 OpDef->getOperand(i).isReg() 2119 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2120 : nullptr; 2121 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2122 return false; 2123 } 2124 return true; 2125 } 2126 } 2127 return false; 2128 } 2129 2130 // Return true if the virtual register represents a constant 2131 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2132 SmallPtrSet<SPIRVType *, 4> Visited; 2133 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2134 return isConstReg(MRI, OpDef, Visited); 2135 return false; 2136 } 2137 2138 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2139 const SPIRVType *ResType, 2140 MachineInstr &I) const { 2141 unsigned N = 0; 2142 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2143 N = GR.getScalarOrVectorComponentCount(ResType); 2144 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2145 N = getArrayComponentCount(MRI, ResType); 2146 else 2147 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2148 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2149 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2150 2151 // check if we may construct a constant vector 2152 bool IsConst = true; 2153 for (unsigned i = I.getNumExplicitDefs(); 2154 i < I.getNumExplicitOperands() && IsConst; ++i) 2155 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2156 IsConst = false; 2157 2158 if (!IsConst && N < 2) 2159 report_fatal_error( 2160 "There must be at least two constituent operands in a vector"); 2161 2162 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2163 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2164 TII.get(IsConst ? SPIRV::OpConstantComposite 2165 : SPIRV::OpCompositeConstruct)) 2166 .addDef(ResVReg) 2167 .addUse(GR.getSPIRVTypeID(ResType)); 2168 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2169 MIB.addUse(I.getOperand(i).getReg()); 2170 return MIB.constrainAllUses(TII, TRI, RBI); 2171 } 2172 2173 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2174 const SPIRVType *ResType, 2175 MachineInstr &I) const { 2176 unsigned N = 0; 2177 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2178 N = GR.getScalarOrVectorComponentCount(ResType); 2179 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2180 N = getArrayComponentCount(MRI, ResType); 2181 else 2182 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2183 2184 unsigned OpIdx = I.getNumExplicitDefs(); 2185 if (!I.getOperand(OpIdx).isReg()) 2186 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2187 2188 // check if we may construct a constant vector 2189 Register OpReg = I.getOperand(OpIdx).getReg(); 2190 bool IsConst = isConstReg(MRI, OpReg); 2191 2192 if (!IsConst && N < 2) 2193 report_fatal_error( 2194 "There must be at least two constituent operands in a vector"); 2195 2196 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2197 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2198 TII.get(IsConst ? SPIRV::OpConstantComposite 2199 : SPIRV::OpCompositeConstruct)) 2200 .addDef(ResVReg) 2201 .addUse(GR.getSPIRVTypeID(ResType)); 2202 for (unsigned i = 0; i < N; ++i) 2203 MIB.addUse(OpReg); 2204 return MIB.constrainAllUses(TII, TRI, RBI); 2205 } 2206 2207 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2208 const SPIRVType *ResType, 2209 MachineInstr &I) const { 2210 2211 unsigned Opcode; 2212 2213 if (STI.canUseExtension( 2214 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2215 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2216 Opcode = SPIRV::OpDemoteToHelperInvocation; 2217 } else { 2218 Opcode = SPIRV::OpKill; 2219 // OpKill must be the last operation of any basic block. 2220 MachineInstr *NextI = I.getNextNode(); 2221 NextI->removeFromParent(); 2222 } 2223 2224 MachineBasicBlock &BB = *I.getParent(); 2225 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2226 .constrainAllUses(TII, TRI, RBI); 2227 } 2228 2229 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2230 const SPIRVType *ResType, 2231 unsigned CmpOpc, 2232 MachineInstr &I) const { 2233 Register Cmp0 = I.getOperand(2).getReg(); 2234 Register Cmp1 = I.getOperand(3).getReg(); 2235 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2236 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2237 "CMP operands should have the same type"); 2238 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2239 .addDef(ResVReg) 2240 .addUse(GR.getSPIRVTypeID(ResType)) 2241 .addUse(Cmp0) 2242 .addUse(Cmp1) 2243 .constrainAllUses(TII, TRI, RBI); 2244 } 2245 2246 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2247 const SPIRVType *ResType, 2248 MachineInstr &I) const { 2249 auto Pred = I.getOperand(1).getPredicate(); 2250 unsigned CmpOpc; 2251 2252 Register CmpOperand = I.getOperand(2).getReg(); 2253 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2254 CmpOpc = getPtrCmpOpcode(Pred); 2255 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2256 CmpOpc = getBoolCmpOpcode(Pred); 2257 else 2258 CmpOpc = getICmpOpcode(Pred); 2259 return selectCmp(ResVReg, ResType, CmpOpc, I); 2260 } 2261 2262 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2263 const MachineInstr &I, 2264 int OpIdx) const { 2265 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2266 "Expected G_FCONSTANT"); 2267 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2268 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2269 } 2270 2271 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2272 const MachineInstr &I, 2273 int OpIdx) const { 2274 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2275 "Expected G_CONSTANT"); 2276 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2277 } 2278 2279 std::pair<Register, bool> 2280 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2281 const SPIRVType *ResType) const { 2282 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2283 const SPIRVType *SpvI32Ty = 2284 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2285 // Find a constant in DT or build a new one. 2286 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2287 Register NewReg = GR.find(ConstInt, GR.CurMF); 2288 bool Result = true; 2289 if (!NewReg.isValid()) { 2290 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2291 GR.add(ConstInt, GR.CurMF, NewReg); 2292 MachineInstr *MI; 2293 MachineBasicBlock &BB = *I.getParent(); 2294 if (Val == 0) { 2295 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2296 .addDef(NewReg) 2297 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2298 } else { 2299 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2300 .addDef(NewReg) 2301 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2302 .addImm(APInt(32, Val).getZExtValue()); 2303 } 2304 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2305 } 2306 return {NewReg, Result}; 2307 } 2308 2309 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2310 const SPIRVType *ResType, 2311 MachineInstr &I) const { 2312 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2313 return selectCmp(ResVReg, ResType, CmpOp, I); 2314 } 2315 2316 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2317 MachineInstr &I) const { 2318 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2319 bool ZeroAsNull = STI.isOpenCLEnv(); 2320 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2321 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2322 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2323 } 2324 2325 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2326 MachineInstr &I) const { 2327 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2328 bool ZeroAsNull = STI.isOpenCLEnv(); 2329 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2330 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2331 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2332 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2333 } 2334 2335 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2336 MachineInstr &I) const { 2337 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2338 bool ZeroAsNull = STI.isOpenCLEnv(); 2339 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2340 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2341 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2342 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2343 } 2344 2345 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2346 const SPIRVType *ResType, 2347 MachineInstr &I) const { 2348 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2349 APInt One = 2350 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2351 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2352 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2353 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2354 } 2355 2356 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2357 const SPIRVType *ResType, 2358 MachineInstr &I, 2359 bool IsSigned) const { 2360 // To extend a bool, we need to use OpSelect between constants. 2361 Register ZeroReg = buildZerosVal(ResType, I); 2362 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2363 bool IsScalarBool = 2364 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2365 unsigned Opcode = 2366 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2367 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2368 .addDef(ResVReg) 2369 .addUse(GR.getSPIRVTypeID(ResType)) 2370 .addUse(I.getOperand(1).getReg()) 2371 .addUse(OneReg) 2372 .addUse(ZeroReg) 2373 .constrainAllUses(TII, TRI, RBI); 2374 } 2375 2376 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2377 const SPIRVType *ResType, 2378 MachineInstr &I, bool IsSigned, 2379 unsigned Opcode) const { 2380 Register SrcReg = I.getOperand(1).getReg(); 2381 // We can convert bool value directly to float type without OpConvert*ToF, 2382 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2383 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2384 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2385 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2386 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2387 const unsigned NumElts = ResType->getOperand(2).getImm(); 2388 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2389 } 2390 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2391 selectSelect(SrcReg, TmpType, I, false); 2392 } 2393 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2394 } 2395 2396 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2397 const SPIRVType *ResType, 2398 MachineInstr &I, bool IsSigned) const { 2399 Register SrcReg = I.getOperand(1).getReg(); 2400 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2401 return selectSelect(ResVReg, ResType, I, IsSigned); 2402 2403 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2404 if (SrcType == ResType) 2405 return BuildCOPY(ResVReg, SrcReg, I); 2406 2407 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2408 return selectUnOp(ResVReg, ResType, I, Opcode); 2409 } 2410 2411 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2412 const SPIRVType *ResType, 2413 MachineInstr &I, 2414 bool IsSigned) const { 2415 MachineIRBuilder MIRBuilder(I); 2416 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2417 MachineBasicBlock &BB = *I.getParent(); 2418 // Ensure we have bool. 2419 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2420 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2421 if (N > 1) 2422 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2423 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2424 // Build less-than-equal and less-than. 2425 // TODO: replace with one-liner createVirtualRegister() from 2426 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2427 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2428 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2429 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2430 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2431 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2432 : SPIRV::OpULessThanEqual)) 2433 .addDef(IsLessEqReg) 2434 .addUse(BoolTypeReg) 2435 .addUse(I.getOperand(1).getReg()) 2436 .addUse(I.getOperand(2).getReg()) 2437 .constrainAllUses(TII, TRI, RBI); 2438 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2439 MRI->setType(IsLessReg, LLT::scalar(64)); 2440 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2441 Result &= BuildMI(BB, I, I.getDebugLoc(), 2442 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2443 .addDef(IsLessReg) 2444 .addUse(BoolTypeReg) 2445 .addUse(I.getOperand(1).getReg()) 2446 .addUse(I.getOperand(2).getReg()) 2447 .constrainAllUses(TII, TRI, RBI); 2448 // Build selects. 2449 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2450 Register NegOneOrZeroReg = 2451 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2452 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2453 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2454 unsigned SelectOpcode = 2455 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2456 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2457 .addDef(NegOneOrZeroReg) 2458 .addUse(ResTypeReg) 2459 .addUse(IsLessReg) 2460 .addUse(buildOnesVal(true, ResType, I)) // -1 2461 .addUse(buildZerosVal(ResType, I)) 2462 .constrainAllUses(TII, TRI, RBI); 2463 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2464 .addDef(ResVReg) 2465 .addUse(ResTypeReg) 2466 .addUse(IsLessEqReg) 2467 .addUse(NegOneOrZeroReg) // -1 or 0 2468 .addUse(buildOnesVal(false, ResType, I)) 2469 .constrainAllUses(TII, TRI, RBI); 2470 } 2471 2472 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2473 Register ResVReg, 2474 MachineInstr &I, 2475 const SPIRVType *IntTy, 2476 const SPIRVType *BoolTy) const { 2477 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2478 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2479 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2480 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2481 Register Zero = buildZerosVal(IntTy, I); 2482 Register One = buildOnesVal(false, IntTy, I); 2483 MachineBasicBlock &BB = *I.getParent(); 2484 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2485 .addDef(BitIntReg) 2486 .addUse(GR.getSPIRVTypeID(IntTy)) 2487 .addUse(IntReg) 2488 .addUse(One) 2489 .constrainAllUses(TII, TRI, RBI); 2490 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2491 .addDef(ResVReg) 2492 .addUse(GR.getSPIRVTypeID(BoolTy)) 2493 .addUse(BitIntReg) 2494 .addUse(Zero) 2495 .constrainAllUses(TII, TRI, RBI); 2496 } 2497 2498 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2499 const SPIRVType *ResType, 2500 MachineInstr &I) const { 2501 Register IntReg = I.getOperand(1).getReg(); 2502 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2503 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2504 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2505 if (ArgType == ResType) 2506 return BuildCOPY(ResVReg, IntReg, I); 2507 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2508 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2509 return selectUnOp(ResVReg, ResType, I, Opcode); 2510 } 2511 2512 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2513 const SPIRVType *ResType, 2514 const APInt &Imm, 2515 MachineInstr &I) const { 2516 unsigned TyOpcode = ResType->getOpcode(); 2517 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2518 MachineBasicBlock &BB = *I.getParent(); 2519 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2520 Imm.isZero()) 2521 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2522 .addDef(ResVReg) 2523 .addUse(GR.getSPIRVTypeID(ResType)) 2524 .constrainAllUses(TII, TRI, RBI); 2525 if (TyOpcode == SPIRV::OpTypeInt) { 2526 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2527 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2528 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2529 } 2530 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2531 .addDef(ResVReg) 2532 .addUse(GR.getSPIRVTypeID(ResType)); 2533 // <=32-bit integers should be caught by the sdag pattern. 2534 assert(Imm.getBitWidth() > 32); 2535 addNumImm(Imm, MIB); 2536 return MIB.constrainAllUses(TII, TRI, RBI); 2537 } 2538 2539 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2540 const SPIRVType *ResType, 2541 MachineInstr &I) const { 2542 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2543 .addDef(ResVReg) 2544 .addUse(GR.getSPIRVTypeID(ResType)) 2545 .constrainAllUses(TII, TRI, RBI); 2546 } 2547 2548 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2549 assert(MO.isReg()); 2550 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2551 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2552 assert(TypeInst->getOperand(1).isReg()); 2553 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2554 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2555 } 2556 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2557 } 2558 2559 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2560 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2561 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2562 return TypeInst->getOperand(2).getImm(); 2563 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2564 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2565 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2566 } 2567 2568 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2569 const SPIRVType *ResType, 2570 MachineInstr &I) const { 2571 MachineBasicBlock &BB = *I.getParent(); 2572 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2573 .addDef(ResVReg) 2574 .addUse(GR.getSPIRVTypeID(ResType)) 2575 // object to insert 2576 .addUse(I.getOperand(3).getReg()) 2577 // composite to insert into 2578 .addUse(I.getOperand(2).getReg()); 2579 for (unsigned i = 4; i < I.getNumOperands(); i++) 2580 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2581 return MIB.constrainAllUses(TII, TRI, RBI); 2582 } 2583 2584 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2585 const SPIRVType *ResType, 2586 MachineInstr &I) const { 2587 MachineBasicBlock &BB = *I.getParent(); 2588 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2589 .addDef(ResVReg) 2590 .addUse(GR.getSPIRVTypeID(ResType)) 2591 .addUse(I.getOperand(2).getReg()); 2592 for (unsigned i = 3; i < I.getNumOperands(); i++) 2593 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2594 return MIB.constrainAllUses(TII, TRI, RBI); 2595 } 2596 2597 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2598 const SPIRVType *ResType, 2599 MachineInstr &I) const { 2600 if (isImm(I.getOperand(4), MRI)) 2601 return selectInsertVal(ResVReg, ResType, I); 2602 MachineBasicBlock &BB = *I.getParent(); 2603 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2604 .addDef(ResVReg) 2605 .addUse(GR.getSPIRVTypeID(ResType)) 2606 .addUse(I.getOperand(2).getReg()) 2607 .addUse(I.getOperand(3).getReg()) 2608 .addUse(I.getOperand(4).getReg()) 2609 .constrainAllUses(TII, TRI, RBI); 2610 } 2611 2612 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2613 const SPIRVType *ResType, 2614 MachineInstr &I) const { 2615 if (isImm(I.getOperand(3), MRI)) 2616 return selectExtractVal(ResVReg, ResType, I); 2617 MachineBasicBlock &BB = *I.getParent(); 2618 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2619 .addDef(ResVReg) 2620 .addUse(GR.getSPIRVTypeID(ResType)) 2621 .addUse(I.getOperand(2).getReg()) 2622 .addUse(I.getOperand(3).getReg()) 2623 .constrainAllUses(TII, TRI, RBI); 2624 } 2625 2626 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2627 const SPIRVType *ResType, 2628 MachineInstr &I) const { 2629 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2630 2631 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2632 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2633 // we have to use Op[InBounds]AccessChain. 2634 const unsigned Opcode = STI.isVulkanEnv() 2635 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2636 : SPIRV::OpAccessChain) 2637 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2638 : SPIRV::OpPtrAccessChain); 2639 2640 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2641 .addDef(ResVReg) 2642 .addUse(GR.getSPIRVTypeID(ResType)) 2643 // Object to get a pointer to. 2644 .addUse(I.getOperand(3).getReg()); 2645 // Adding indices. 2646 const unsigned StartingIndex = 2647 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2648 ? 5 2649 : 4; 2650 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2651 Res.addUse(I.getOperand(i).getReg()); 2652 return Res.constrainAllUses(TII, TRI, RBI); 2653 } 2654 2655 // Maybe wrap a value into OpSpecConstantOp 2656 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2657 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2658 bool Result = true; 2659 unsigned Lim = I.getNumExplicitOperands(); 2660 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2661 Register OpReg = I.getOperand(i).getReg(); 2662 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2663 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2664 SmallPtrSet<SPIRVType *, 4> Visited; 2665 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2666 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2667 GR.isAggregateType(OpType)) { 2668 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2669 // by selectAddrSpaceCast() 2670 CompositeArgs.push_back(OpReg); 2671 continue; 2672 } 2673 MachineFunction *MF = I.getMF(); 2674 Register WrapReg = GR.find(OpDefine, MF); 2675 if (WrapReg.isValid()) { 2676 CompositeArgs.push_back(WrapReg); 2677 continue; 2678 } 2679 // Create a new register for the wrapper 2680 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2681 GR.add(OpDefine, MF, WrapReg); 2682 CompositeArgs.push_back(WrapReg); 2683 // Decorate the wrapper register and generate a new instruction 2684 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2685 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2686 MachineBasicBlock &BB = *I.getParent(); 2687 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2688 .addDef(WrapReg) 2689 .addUse(GR.getSPIRVTypeID(OpType)) 2690 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2691 .addUse(OpReg) 2692 .constrainAllUses(TII, TRI, RBI); 2693 if (!Result) 2694 break; 2695 } 2696 return Result; 2697 } 2698 2699 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2700 const SPIRVType *ResType, 2701 MachineInstr &I) const { 2702 MachineBasicBlock &BB = *I.getParent(); 2703 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2704 switch (IID) { 2705 case Intrinsic::spv_load: 2706 return selectLoad(ResVReg, ResType, I); 2707 case Intrinsic::spv_store: 2708 return selectStore(I); 2709 case Intrinsic::spv_extractv: 2710 return selectExtractVal(ResVReg, ResType, I); 2711 case Intrinsic::spv_insertv: 2712 return selectInsertVal(ResVReg, ResType, I); 2713 case Intrinsic::spv_extractelt: 2714 return selectExtractElt(ResVReg, ResType, I); 2715 case Intrinsic::spv_insertelt: 2716 return selectInsertElt(ResVReg, ResType, I); 2717 case Intrinsic::spv_gep: 2718 return selectGEP(ResVReg, ResType, I); 2719 case Intrinsic::spv_unref_global: 2720 case Intrinsic::spv_init_global: { 2721 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2722 MachineInstr *Init = I.getNumExplicitOperands() > 2 2723 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2724 : nullptr; 2725 assert(MI); 2726 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2727 } 2728 case Intrinsic::spv_undef: { 2729 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2730 .addDef(ResVReg) 2731 .addUse(GR.getSPIRVTypeID(ResType)); 2732 return MIB.constrainAllUses(TII, TRI, RBI); 2733 } 2734 case Intrinsic::spv_const_composite: { 2735 // If no values are attached, the composite is null constant. 2736 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2737 // Select a proper instruction. 2738 unsigned Opcode = SPIRV::OpConstantNull; 2739 SmallVector<Register> CompositeArgs; 2740 if (!IsNull) { 2741 Opcode = SPIRV::OpConstantComposite; 2742 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2743 return false; 2744 } 2745 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2746 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2747 .addDef(ResVReg) 2748 .addUse(GR.getSPIRVTypeID(ResType)); 2749 // skip type MD node we already used when generated assign.type for this 2750 if (!IsNull) { 2751 for (Register OpReg : CompositeArgs) 2752 MIB.addUse(OpReg); 2753 } 2754 return MIB.constrainAllUses(TII, TRI, RBI); 2755 } 2756 case Intrinsic::spv_assign_name: { 2757 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2758 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2759 for (unsigned i = I.getNumExplicitDefs() + 2; 2760 i < I.getNumExplicitOperands(); ++i) { 2761 MIB.addImm(I.getOperand(i).getImm()); 2762 } 2763 return MIB.constrainAllUses(TII, TRI, RBI); 2764 } 2765 case Intrinsic::spv_switch: { 2766 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2767 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2768 if (I.getOperand(i).isReg()) 2769 MIB.addReg(I.getOperand(i).getReg()); 2770 else if (I.getOperand(i).isCImm()) 2771 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2772 else if (I.getOperand(i).isMBB()) 2773 MIB.addMBB(I.getOperand(i).getMBB()); 2774 else 2775 llvm_unreachable("Unexpected OpSwitch operand"); 2776 } 2777 return MIB.constrainAllUses(TII, TRI, RBI); 2778 } 2779 case Intrinsic::spv_loop_merge: 2780 case Intrinsic::spv_selection_merge: { 2781 const auto Opcode = IID == Intrinsic::spv_selection_merge 2782 ? SPIRV::OpSelectionMerge 2783 : SPIRV::OpLoopMerge; 2784 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2785 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2786 assert(I.getOperand(i).isMBB()); 2787 MIB.addMBB(I.getOperand(i).getMBB()); 2788 } 2789 MIB.addImm(SPIRV::SelectionControl::None); 2790 return MIB.constrainAllUses(TII, TRI, RBI); 2791 } 2792 case Intrinsic::spv_cmpxchg: 2793 return selectAtomicCmpXchg(ResVReg, ResType, I); 2794 case Intrinsic::spv_unreachable: 2795 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2796 .constrainAllUses(TII, TRI, RBI); 2797 case Intrinsic::spv_alloca: 2798 return selectFrameIndex(ResVReg, ResType, I); 2799 case Intrinsic::spv_alloca_array: 2800 return selectAllocaArray(ResVReg, ResType, I); 2801 case Intrinsic::spv_assume: 2802 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2803 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2804 .addUse(I.getOperand(1).getReg()) 2805 .constrainAllUses(TII, TRI, RBI); 2806 break; 2807 case Intrinsic::spv_expect: 2808 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2809 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2810 .addDef(ResVReg) 2811 .addUse(GR.getSPIRVTypeID(ResType)) 2812 .addUse(I.getOperand(2).getReg()) 2813 .addUse(I.getOperand(3).getReg()) 2814 .constrainAllUses(TII, TRI, RBI); 2815 break; 2816 case Intrinsic::arithmetic_fence: 2817 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2818 return BuildMI(BB, I, I.getDebugLoc(), 2819 TII.get(SPIRV::OpArithmeticFenceEXT)) 2820 .addDef(ResVReg) 2821 .addUse(GR.getSPIRVTypeID(ResType)) 2822 .addUse(I.getOperand(2).getReg()) 2823 .constrainAllUses(TII, TRI, RBI); 2824 else 2825 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2826 break; 2827 case Intrinsic::spv_thread_id: 2828 return selectSpvThreadId(ResVReg, ResType, I); 2829 case Intrinsic::spv_fdot: 2830 return selectFloatDot(ResVReg, ResType, I); 2831 case Intrinsic::spv_udot: 2832 case Intrinsic::spv_sdot: 2833 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2834 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2835 return selectIntegerDot(ResVReg, ResType, I, 2836 /*Signed=*/IID == Intrinsic::spv_sdot); 2837 return selectIntegerDotExpansion(ResVReg, ResType, I); 2838 case Intrinsic::spv_dot4add_i8packed: 2839 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2840 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2841 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2842 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2843 case Intrinsic::spv_dot4add_u8packed: 2844 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2845 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2846 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2847 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2848 case Intrinsic::spv_all: 2849 return selectAll(ResVReg, ResType, I); 2850 case Intrinsic::spv_any: 2851 return selectAny(ResVReg, ResType, I); 2852 case Intrinsic::spv_cross: 2853 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2854 case Intrinsic::spv_lerp: 2855 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2856 case Intrinsic::spv_length: 2857 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2858 case Intrinsic::spv_degrees: 2859 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2860 case Intrinsic::spv_frac: 2861 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2862 case Intrinsic::spv_normalize: 2863 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2864 case Intrinsic::spv_rsqrt: 2865 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2866 case Intrinsic::spv_sign: 2867 return selectSign(ResVReg, ResType, I); 2868 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2869 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2870 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2871 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2872 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2873 bool Result = true; 2874 auto MemSemConstant = 2875 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2876 Register MemSemReg = MemSemConstant.first; 2877 Result &= MemSemConstant.second; 2878 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2879 Register ScopeReg = ScopeConstant.first; 2880 Result &= ScopeConstant.second; 2881 MachineBasicBlock &BB = *I.getParent(); 2882 return Result && 2883 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2884 .addUse(ScopeReg) 2885 .addUse(ScopeReg) 2886 .addUse(MemSemReg) 2887 .constrainAllUses(TII, TRI, RBI); 2888 } 2889 case Intrinsic::spv_lifetime_start: 2890 case Intrinsic::spv_lifetime_end: { 2891 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2892 : SPIRV::OpLifetimeStop; 2893 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2894 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2895 if (Size == -1) 2896 Size = 0; 2897 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2898 .addUse(PtrReg) 2899 .addImm(Size) 2900 .constrainAllUses(TII, TRI, RBI); 2901 } 2902 case Intrinsic::spv_saturate: 2903 return selectSaturate(ResVReg, ResType, I); 2904 case Intrinsic::spv_nclamp: 2905 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2906 case Intrinsic::spv_uclamp: 2907 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2908 case Intrinsic::spv_sclamp: 2909 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2910 case Intrinsic::spv_wave_active_countbits: 2911 return selectWaveActiveCountBits(ResVReg, ResType, I); 2912 case Intrinsic::spv_wave_any: 2913 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 2914 case Intrinsic::spv_wave_is_first_lane: 2915 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 2916 case Intrinsic::spv_wave_readlane: 2917 return selectWaveOpInst(ResVReg, ResType, I, 2918 SPIRV::OpGroupNonUniformShuffle); 2919 case Intrinsic::spv_step: 2920 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2921 case Intrinsic::spv_radians: 2922 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2923 // Discard intrinsics which we do not expect to actually represent code after 2924 // lowering or intrinsics which are not implemented but should not crash when 2925 // found in a customer's LLVM IR input. 2926 case Intrinsic::instrprof_increment: 2927 case Intrinsic::instrprof_increment_step: 2928 case Intrinsic::instrprof_value_profile: 2929 break; 2930 // Discard internal intrinsics. 2931 case Intrinsic::spv_value_md: 2932 break; 2933 case Intrinsic::spv_handle_fromBinding: { 2934 return selectHandleFromBinding(ResVReg, ResType, I); 2935 } 2936 case Intrinsic::spv_typedBufferStore: { 2937 selectImageWriteIntrinsic(I); 2938 return true; 2939 } 2940 case Intrinsic::spv_typedBufferLoad: { 2941 selectReadImageIntrinsic(ResVReg, ResType, I); 2942 return true; 2943 } 2944 case Intrinsic::spv_discard: { 2945 return selectDiscard(ResVReg, ResType, I); 2946 } 2947 default: { 2948 std::string DiagMsg; 2949 raw_string_ostream OS(DiagMsg); 2950 I.print(OS); 2951 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2952 report_fatal_error(DiagMsg.c_str(), false); 2953 } 2954 } 2955 return true; 2956 } 2957 2958 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2959 const SPIRVType *ResType, 2960 MachineInstr &I) const { 2961 2962 uint32_t Set = foldImm(I.getOperand(2), MRI); 2963 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2964 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2965 Register IndexReg = I.getOperand(5).getReg(); 2966 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2967 2968 MachineIRBuilder MIRBuilder(I); 2969 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2970 IndexReg, IsNonUniform, MIRBuilder); 2971 2972 if (IsNonUniform) 2973 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2974 2975 // TODO: For now we assume the resource is an image, which needs to be 2976 // loaded to get the handle. That will not be true for storage buffers. 2977 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2978 .addDef(ResVReg) 2979 .addUse(GR.getSPIRVTypeID(ResType)) 2980 .addUse(VarReg) 2981 .constrainAllUses(TII, TRI, RBI); 2982 } 2983 2984 void SPIRVInstructionSelector::selectReadImageIntrinsic( 2985 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2986 2987 // If the load of the image is in a different basic block, then 2988 // this will generate invalid code. A proper solution is to move 2989 // the OpLoad from selectHandleFromBinding here. However, to do 2990 // that we will need to change the return type of the intrinsic. 2991 // We will do that when we can, but for now trying to move forward with other 2992 // issues. 2993 Register ImageReg = I.getOperand(2).getReg(); 2994 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 2995 "The image must be loaded in the same basic block as its use."); 2996 2997 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2998 if (ResultSize == 4) { 2999 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3000 .addDef(ResVReg) 3001 .addUse(GR.getSPIRVTypeID(ResType)) 3002 .addUse(ImageReg) 3003 .addUse(I.getOperand(3).getReg()); 3004 return; 3005 } 3006 3007 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 3008 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3009 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3010 .addDef(ReadReg) 3011 .addUse(GR.getSPIRVTypeID(ReadType)) 3012 .addUse(ImageReg) 3013 .addUse(I.getOperand(3).getReg()); 3014 3015 if (ResultSize == 1) { 3016 BuildMI(*I.getParent(), I, I.getDebugLoc(), 3017 TII.get(SPIRV::OpCompositeExtract)) 3018 .addDef(ResVReg) 3019 .addUse(GR.getSPIRVTypeID(ResType)) 3020 .addUse(ReadReg) 3021 .addImm(0); 3022 return; 3023 } 3024 extractSubvector(ResVReg, ResType, ReadReg, I); 3025 } 3026 3027 void SPIRVInstructionSelector::extractSubvector( 3028 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3029 MachineInstr &InsertionPoint) const { 3030 SPIRVType *InputType = GR.getResultType(ReadReg); 3031 [[maybe_unused]] uint64_t InputSize = 3032 GR.getScalarOrVectorComponentCount(InputType); 3033 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3034 assert(InputSize > 1 && "The input must be a vector."); 3035 assert(ResultSize > 1 && "The result must be a vector."); 3036 assert(ResultSize < InputSize && 3037 "Cannot extract more element than there are in the input."); 3038 SmallVector<Register> ComponentRegisters; 3039 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3040 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3041 for (uint64_t I = 0; I < ResultSize; I++) { 3042 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3043 BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3044 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3045 .addDef(ComponentReg) 3046 .addUse(ScalarType->getOperand(0).getReg()) 3047 .addUse(ReadReg) 3048 .addImm(I); 3049 ComponentRegisters.emplace_back(ComponentReg); 3050 } 3051 3052 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3053 InsertionPoint.getDebugLoc(), 3054 TII.get(SPIRV::OpCompositeConstruct)) 3055 .addDef(ResVReg) 3056 .addUse(GR.getSPIRVTypeID(ResType)); 3057 3058 for (Register ComponentReg : ComponentRegisters) 3059 MIB.addUse(ComponentReg); 3060 } 3061 3062 void SPIRVInstructionSelector::selectImageWriteIntrinsic( 3063 MachineInstr &I) const { 3064 // If the load of the image is in a different basic block, then 3065 // this will generate invalid code. A proper solution is to move 3066 // the OpLoad from selectHandleFromBinding here. However, to do 3067 // that we will need to change the return type of the intrinsic. 3068 // We will do that when we can, but for now trying to move forward with other 3069 // issues. 3070 Register ImageReg = I.getOperand(1).getReg(); 3071 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3072 "The image must be loaded in the same basic block as its use."); 3073 Register CoordinateReg = I.getOperand(2).getReg(); 3074 Register DataReg = I.getOperand(3).getReg(); 3075 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3076 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3077 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite)) 3078 .addUse(ImageReg) 3079 .addUse(CoordinateReg) 3080 .addUse(DataReg); 3081 } 3082 3083 Register SPIRVInstructionSelector::buildPointerToResource( 3084 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3085 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3086 MachineIRBuilder MIRBuilder) const { 3087 if (ArraySize == 1) 3088 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3089 MIRBuilder); 3090 3091 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3092 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3093 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3094 VarType, Set, Binding, MIRBuilder); 3095 3096 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3097 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3098 3099 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3100 if (IsNonUniform) { 3101 // It is unclear which value needs to be marked an non-uniform, so both 3102 // the index and the access changed are decorated as non-uniform. 3103 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3104 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3105 } 3106 3107 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3108 .addDef(AcReg) 3109 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3110 .addUse(VarReg) 3111 .addUse(IndexReg); 3112 3113 return AcReg; 3114 } 3115 3116 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3117 const SPIRVType *ResType, 3118 MachineInstr &I, 3119 bool IsSigned) const { 3120 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3121 // zero or sign extend 3122 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3123 bool Result = 3124 selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, Opcode); 3125 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3126 } 3127 3128 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3129 const SPIRVType *ResType, 3130 MachineInstr &I, 3131 Register SrcReg, 3132 bool IsSigned) const { 3133 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3134 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3135 .addDef(ResVReg) 3136 .addUse(GR.getSPIRVTypeID(ResType)) 3137 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3138 .addImm(Opcode) 3139 .addUse(SrcReg) 3140 .constrainAllUses(TII, TRI, RBI); 3141 } 3142 3143 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3144 const SPIRVType *ResType, 3145 MachineInstr &I, 3146 bool IsSigned) const { 3147 Register OpReg = I.getOperand(2).getReg(); 3148 // 1. split our int64 into 2 pieces using a bitcast 3149 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3150 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3151 MachineIRBuilder MIRBuilder(I); 3152 SPIRVType *postCastT = 3153 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3154 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3155 bool Result = 3156 selectOpWithSrcs(bitcastReg, postCastT, I, {OpReg}, SPIRV::OpBitcast); 3157 3158 // 2. call firstbithigh 3159 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3160 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3161 3162 // 3. split result vector into high bits and low bits 3163 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3164 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3165 3166 bool ZeroAsNull = STI.isOpenCLEnv(); 3167 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3168 if (isScalarRes) { 3169 // if scalar do a vector extract 3170 Result &= selectOpWithSrcs( 3171 HighReg, ResType, I, 3172 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3173 SPIRV::OpVectorExtractDynamic); 3174 Result &= selectOpWithSrcs( 3175 LowReg, ResType, I, 3176 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3177 SPIRV::OpVectorExtractDynamic); 3178 } else { // vector case do a shufflevector 3179 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3180 TII.get(SPIRV::OpVectorShuffle)) 3181 .addDef(HighReg) 3182 .addUse(GR.getSPIRVTypeID(ResType)) 3183 .addUse(FBHReg) 3184 .addUse(FBHReg); 3185 // ^^ this vector will not be selected from; could be empty 3186 unsigned j; 3187 for (j = 0; j < count * 2; j += 2) { 3188 MIB.addImm(j); 3189 } 3190 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3191 3192 // get low bits 3193 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3194 TII.get(SPIRV::OpVectorShuffle)) 3195 .addDef(LowReg) 3196 .addUse(GR.getSPIRVTypeID(ResType)) 3197 .addUse(FBHReg) 3198 .addUse(FBHReg); 3199 // ^^ this vector will not be selected from; could be empty 3200 for (j = 1; j < count * 2; j += 2) { 3201 MIB.addImm(j); 3202 } 3203 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3204 } 3205 3206 // 4. check if result of each top 32 bits is == -1 3207 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3208 Register NegOneReg; 3209 Register Reg0; 3210 Register Reg32; 3211 unsigned selectOp; 3212 unsigned addOp; 3213 if (isScalarRes) { 3214 NegOneReg = 3215 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3216 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3217 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3218 selectOp = SPIRV::OpSelectSISCond; 3219 addOp = SPIRV::OpIAddS; 3220 } else { 3221 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3222 NegOneReg = 3223 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3224 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3225 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3226 selectOp = SPIRV::OpSelectVIVCond; 3227 addOp = SPIRV::OpIAddV; 3228 } 3229 3230 // check if the high bits are == -1; true if -1 3231 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3232 Result &= selectOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3233 SPIRV::OpIEqual); 3234 3235 // Select low bits if true in BReg, otherwise high bits 3236 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3237 Result &= 3238 selectOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, selectOp); 3239 3240 // Add 32 for high bits, 0 for low bits 3241 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3242 Result &= selectOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3243 3244 return Result && 3245 selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3246 } 3247 3248 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3249 const SPIRVType *ResType, 3250 MachineInstr &I, 3251 bool IsSigned) const { 3252 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3253 Register OpReg = I.getOperand(2).getReg(); 3254 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3255 3256 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3257 case 16: 3258 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3259 case 32: 3260 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3261 case 64: 3262 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3263 default: 3264 report_fatal_error( 3265 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3266 } 3267 } 3268 3269 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3270 const SPIRVType *ResType, 3271 MachineInstr &I) const { 3272 // there was an allocation size parameter to the allocation instruction 3273 // that is not 1 3274 MachineBasicBlock &BB = *I.getParent(); 3275 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3276 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3277 .addDef(ResVReg) 3278 .addUse(GR.getSPIRVTypeID(ResType)) 3279 .addUse(I.getOperand(2).getReg()) 3280 .constrainAllUses(TII, TRI, RBI); 3281 if (!STI.isVulkanEnv()) { 3282 unsigned Alignment = I.getOperand(3).getImm(); 3283 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3284 } 3285 return Res; 3286 } 3287 3288 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3289 const SPIRVType *ResType, 3290 MachineInstr &I) const { 3291 // Change order of instructions if needed: all OpVariable instructions in a 3292 // function must be the first instructions in the first block 3293 auto It = getOpVariableMBBIt(I); 3294 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3295 TII.get(SPIRV::OpVariable)) 3296 .addDef(ResVReg) 3297 .addUse(GR.getSPIRVTypeID(ResType)) 3298 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3299 .constrainAllUses(TII, TRI, RBI); 3300 if (!STI.isVulkanEnv()) { 3301 unsigned Alignment = I.getOperand(2).getImm(); 3302 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3303 {Alignment}); 3304 } 3305 return Res; 3306 } 3307 3308 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3309 // InstructionSelector walks backwards through the instructions. We can use 3310 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3311 // first, so can generate an OpBranchConditional here. If there is no 3312 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3313 const MachineInstr *PrevI = I.getPrevNode(); 3314 MachineBasicBlock &MBB = *I.getParent(); 3315 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3316 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3317 .addUse(PrevI->getOperand(0).getReg()) 3318 .addMBB(PrevI->getOperand(1).getMBB()) 3319 .addMBB(I.getOperand(0).getMBB()) 3320 .constrainAllUses(TII, TRI, RBI); 3321 } 3322 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3323 .addMBB(I.getOperand(0).getMBB()) 3324 .constrainAllUses(TII, TRI, RBI); 3325 } 3326 3327 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3328 // InstructionSelector walks backwards through the instructions. For an 3329 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3330 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3331 // generate the OpBranchConditional in selectBranch above. 3332 // 3333 // If an OpBranchConditional has been generated, we simply return, as the work 3334 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3335 // implicit fallthrough to the next basic block, so we need to create an 3336 // OpBranchConditional with an explicit "false" argument pointing to the next 3337 // basic block that LLVM would fall through to. 3338 const MachineInstr *NextI = I.getNextNode(); 3339 // Check if this has already been successfully selected. 3340 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3341 return true; 3342 // Must be relying on implicit block fallthrough, so generate an 3343 // OpBranchConditional with the "next" basic block as the "false" target. 3344 MachineBasicBlock &MBB = *I.getParent(); 3345 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3346 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3347 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3348 .addUse(I.getOperand(0).getReg()) 3349 .addMBB(I.getOperand(1).getMBB()) 3350 .addMBB(NextMBB) 3351 .constrainAllUses(TII, TRI, RBI); 3352 } 3353 3354 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3355 const SPIRVType *ResType, 3356 MachineInstr &I) const { 3357 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3358 .addDef(ResVReg) 3359 .addUse(GR.getSPIRVTypeID(ResType)); 3360 const unsigned NumOps = I.getNumOperands(); 3361 for (unsigned i = 1; i < NumOps; i += 2) { 3362 MIB.addUse(I.getOperand(i + 0).getReg()); 3363 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3364 } 3365 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3366 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3367 MIB->removeOperand(1); 3368 return Res; 3369 } 3370 3371 bool SPIRVInstructionSelector::selectGlobalValue( 3372 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3373 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3374 MachineIRBuilder MIRBuilder(I); 3375 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3376 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3377 SPIRVType *PointerBaseType; 3378 if (GVType->isArrayTy()) { 3379 SPIRVType *ArrayElementType = 3380 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3381 SPIRV::AccessQualifier::ReadWrite, false); 3382 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3383 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3384 } else { 3385 PointerBaseType = GR.getOrCreateSPIRVType( 3386 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3387 } 3388 3389 std::string GlobalIdent; 3390 if (!GV->hasName()) { 3391 unsigned &ID = UnnamedGlobalIDs[GV]; 3392 if (ID == 0) 3393 ID = UnnamedGlobalIDs.size(); 3394 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3395 } else { 3396 GlobalIdent = GV->getGlobalIdentifier(); 3397 } 3398 3399 // Behaviour of functions as operands depends on availability of the 3400 // corresponding extension (SPV_INTEL_function_pointers): 3401 // - If there is an extension to operate with functions as operands: 3402 // We create a proper constant operand and evaluate a correct type for a 3403 // function pointer. 3404 // - Without the required extension: 3405 // We have functions as operands in tests with blocks of instruction e.g. in 3406 // transcoding/global_block.ll. These operands are not used and should be 3407 // substituted by zero constants. Their type is expected to be always 3408 // OpTypePointer Function %uchar. 3409 if (isa<Function>(GV)) { 3410 const Constant *ConstVal = GV; 3411 MachineBasicBlock &BB = *I.getParent(); 3412 Register NewReg = GR.find(ConstVal, GR.CurMF); 3413 if (!NewReg.isValid()) { 3414 Register NewReg = ResVReg; 3415 GR.add(ConstVal, GR.CurMF, NewReg); 3416 const Function *GVFun = 3417 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3418 ? dyn_cast<Function>(GV) 3419 : nullptr; 3420 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3421 PointerBaseType, I, TII, 3422 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3423 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3424 if (GVFun) { 3425 // References to a function via function pointers generate virtual 3426 // registers without a definition. We will resolve it later, during 3427 // module analysis stage. 3428 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3429 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3430 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3431 MachineInstrBuilder MB = 3432 BuildMI(BB, I, I.getDebugLoc(), 3433 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3434 .addDef(NewReg) 3435 .addUse(GR.getSPIRVTypeID(ResType)) 3436 .addUse(FuncVReg); 3437 // mapping the function pointer to the used Function 3438 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3439 return MB.constrainAllUses(TII, TRI, RBI); 3440 } 3441 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3442 .addDef(NewReg) 3443 .addUse(GR.getSPIRVTypeID(ResType)) 3444 .constrainAllUses(TII, TRI, RBI); 3445 } 3446 assert(NewReg != ResVReg); 3447 return BuildCOPY(ResVReg, NewReg, I); 3448 } 3449 auto GlobalVar = cast<GlobalVariable>(GV); 3450 assert(GlobalVar->getName() != "llvm.global.annotations"); 3451 3452 bool HasInit = GlobalVar->hasInitializer() && 3453 !isa<UndefValue>(GlobalVar->getInitializer()); 3454 // Skip empty declaration for GVs with initilaizers till we get the decl with 3455 // passed initializer. 3456 if (HasInit && !Init) 3457 return true; 3458 3459 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage; 3460 SPIRV::LinkageType::LinkageType LnkType = 3461 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3462 ? SPIRV::LinkageType::Import 3463 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3464 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3465 ? SPIRV::LinkageType::LinkOnceODR 3466 : SPIRV::LinkageType::Export); 3467 3468 const unsigned AddrSpace = GV->getAddressSpace(); 3469 SPIRV::StorageClass::StorageClass StorageClass = 3470 addressSpaceToStorageClass(AddrSpace, STI); 3471 SPIRVType *ResType = 3472 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3473 Register Reg = GR.buildGlobalVariable( 3474 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3475 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3476 return Reg.isValid(); 3477 } 3478 3479 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3480 const SPIRVType *ResType, 3481 MachineInstr &I) const { 3482 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3483 return selectExtInst(ResVReg, ResType, I, CL::log10); 3484 } 3485 3486 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3487 // is implemented as: 3488 // log10(x) = log2(x) * (1 / log2(10)) 3489 // = log2(x) * 0.30103 3490 3491 MachineIRBuilder MIRBuilder(I); 3492 MachineBasicBlock &BB = *I.getParent(); 3493 3494 // Build log2(x). 3495 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3496 bool Result = 3497 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3498 .addDef(VarReg) 3499 .addUse(GR.getSPIRVTypeID(ResType)) 3500 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3501 .addImm(GL::Log2) 3502 .add(I.getOperand(1)) 3503 .constrainAllUses(TII, TRI, RBI); 3504 3505 // Build 0.30103. 3506 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3507 ResType->getOpcode() == SPIRV::OpTypeFloat); 3508 // TODO: Add matrix implementation once supported by the HLSL frontend. 3509 const SPIRVType *SpirvScalarType = 3510 ResType->getOpcode() == SPIRV::OpTypeVector 3511 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3512 : ResType; 3513 Register ScaleReg = 3514 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3515 3516 // Multiply log2(x) by 0.30103 to get log10(x) result. 3517 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3518 ? SPIRV::OpVectorTimesScalar 3519 : SPIRV::OpFMulS; 3520 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3521 .addDef(ResVReg) 3522 .addUse(GR.getSPIRVTypeID(ResType)) 3523 .addUse(VarReg) 3524 .addUse(ScaleReg) 3525 .constrainAllUses(TII, TRI, RBI); 3526 } 3527 3528 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3529 const SPIRVType *ResType, 3530 MachineInstr &I) const { 3531 // DX intrinsic: @llvm.dx.thread.id(i32) 3532 // ID Name Description 3533 // 93 ThreadId reads the thread ID 3534 3535 MachineIRBuilder MIRBuilder(I); 3536 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3537 const SPIRVType *Vec3Ty = 3538 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3539 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3540 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3541 3542 // Create new register for GlobalInvocationID builtin variable. 3543 Register NewRegister = 3544 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3545 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3546 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3547 3548 // Build GlobalInvocationID global variable with the necessary decorations. 3549 Register Variable = GR.buildGlobalVariable( 3550 NewRegister, PtrType, 3551 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3552 SPIRV::StorageClass::Input, nullptr, true, true, 3553 SPIRV::LinkageType::Import, MIRBuilder, false); 3554 3555 // Create new register for loading value. 3556 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3557 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3558 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3559 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3560 3561 // Load v3uint value from the global variable. 3562 bool Result = 3563 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3564 .addDef(LoadedRegister) 3565 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3566 .addUse(Variable); 3567 3568 // Get Thread ID index. Expecting operand is a constant immediate value, 3569 // wrapped in a type assignment. 3570 assert(I.getOperand(2).isReg()); 3571 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3572 3573 // Extract the thread ID from the loaded vector value. 3574 MachineBasicBlock &BB = *I.getParent(); 3575 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3576 .addDef(ResVReg) 3577 .addUse(GR.getSPIRVTypeID(ResType)) 3578 .addUse(LoadedRegister) 3579 .addImm(ThreadId); 3580 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3581 } 3582 3583 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3584 MachineInstr &I) const { 3585 MachineIRBuilder MIRBuilder(I); 3586 if (Type->getOpcode() != SPIRV::OpTypeVector) 3587 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3588 3589 uint64_t VectorSize = Type->getOperand(2).getImm(); 3590 if (VectorSize == 4) 3591 return Type; 3592 3593 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3594 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3595 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3596 } 3597 3598 namespace llvm { 3599 InstructionSelector * 3600 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3601 const SPIRVSubtarget &Subtarget, 3602 const RegisterBankInfo &RBI) { 3603 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3604 } 3605 } // namespace llvm 3606