1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 37 #define DEBUG_TYPE "spirv-isel" 38 39 using namespace llvm; 40 namespace CL = SPIRV::OpenCLExtInst; 41 namespace GL = SPIRV::GLSLExtInst; 42 43 using ExtInstList = 44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 45 46 namespace { 47 48 #define GET_GLOBALISEL_PREDICATE_BITSET 49 #include "SPIRVGenGlobalISel.inc" 50 #undef GET_GLOBALISEL_PREDICATE_BITSET 51 52 class SPIRVInstructionSelector : public InstructionSelector { 53 const SPIRVSubtarget &STI; 54 const SPIRVInstrInfo &TII; 55 const SPIRVRegisterInfo &TRI; 56 const RegisterBankInfo &RBI; 57 SPIRVGlobalRegistry &GR; 58 MachineRegisterInfo *MRI; 59 MachineFunction *HasVRegsReset = nullptr; 60 61 /// We need to keep track of the number we give to anonymous global values to 62 /// generate the same name every time when this is needed. 63 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 64 SmallPtrSet<MachineInstr *, 8> DeadMIs; 65 66 public: 67 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 68 const SPIRVSubtarget &ST, 69 const RegisterBankInfo &RBI); 70 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 71 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 72 BlockFrequencyInfo *BFI) override; 73 // Common selection code. Instruction-specific selection occurs in spvSelect. 74 bool select(MachineInstr &I) override; 75 static const char *getName() { return DEBUG_TYPE; } 76 77 #define GET_GLOBALISEL_PREDICATES_DECL 78 #include "SPIRVGenGlobalISel.inc" 79 #undef GET_GLOBALISEL_PREDICATES_DECL 80 81 #define GET_GLOBALISEL_TEMPORARIES_DECL 82 #include "SPIRVGenGlobalISel.inc" 83 #undef GET_GLOBALISEL_TEMPORARIES_DECL 84 85 private: 86 void resetVRegsType(MachineFunction &MF); 87 88 // tblgen-erated 'select' implementation, used as the initial selector for 89 // the patterns that don't require complex C++. 90 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 91 92 // All instruction-specific selection that didn't happen in "select()". 93 // Is basically a large Switch/Case delegating to all other select method. 94 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 95 MachineInstr &I) const; 96 97 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 98 MachineInstr &I, bool IsSigned) const; 99 100 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 101 MachineInstr &I, bool IsSigned) const; 102 103 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 104 MachineInstr &I, Register SrcReg, 105 bool IsSigned) const; 106 107 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 108 MachineInstr &I, bool IsSigned) const; 109 110 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 111 const MachineInstr *Init = nullptr) const; 112 113 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 114 MachineInstr &I, std::vector<Register> SrcRegs, 115 unsigned Opcode) const; 116 117 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 118 unsigned Opcode) const; 119 120 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 121 MachineInstr &I) const; 122 123 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I) const; 125 bool selectStore(MachineInstr &I) const; 126 127 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I) const; 129 bool selectStackRestore(MachineInstr &I) const; 130 131 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 132 133 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 134 MachineInstr &I, unsigned NewOpcode, 135 unsigned NegateOpcode = 0) const; 136 137 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 138 MachineInstr &I) const; 139 140 bool selectFence(MachineInstr &I) const; 141 142 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 143 MachineInstr &I) const; 144 145 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I, unsigned OpType) const; 147 148 bool selectAll(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 151 bool selectAny(Register ResVReg, const SPIRVType *ResType, 152 MachineInstr &I) const; 153 154 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 155 MachineInstr &I) const; 156 157 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 158 MachineInstr &I) const; 159 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 162 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 163 unsigned comparisonOpcode, MachineInstr &I) const; 164 bool selectCross(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 167 MachineInstr &I) const; 168 169 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I) const; 171 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 172 MachineInstr &I) const; 173 174 bool selectSign(Register ResVReg, const SPIRVType *ResType, 175 MachineInstr &I) const; 176 177 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 178 MachineInstr &I) const; 179 180 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 181 MachineInstr &I, unsigned Opcode) const; 182 183 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 184 MachineInstr &I, bool Signed) const; 185 186 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 189 template <bool Signed> 190 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 191 MachineInstr &I) const; 192 template <bool Signed> 193 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 194 MachineInstr &I) const; 195 196 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 197 int OpIdx) const; 198 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 199 int OpIdx) const; 200 201 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 202 MachineInstr &I) const; 203 204 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 205 bool IsSigned) const; 206 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 207 bool IsSigned, unsigned Opcode) const; 208 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 209 bool IsSigned) const; 210 211 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 212 MachineInstr &I) const; 213 214 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 215 bool IsSigned) const; 216 217 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 218 const SPIRVType *intTy, const SPIRVType *boolTy) const; 219 220 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I) const; 224 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 227 MachineInstr &I) const; 228 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 229 MachineInstr &I) const; 230 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 233 MachineInstr &I) const; 234 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 235 MachineInstr &I) const; 236 237 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 240 MachineInstr &I) const; 241 242 bool selectBranch(MachineInstr &I) const; 243 bool selectBranchCond(MachineInstr &I) const; 244 245 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 246 MachineInstr &I) const; 247 248 [[maybe_unused]] bool selectExtInst(Register ResVReg, 249 const SPIRVType *RestType, 250 MachineInstr &I, 251 GL::GLSLExtInst GLInst) const; 252 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 253 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 254 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I, CL::OpenCLExtInst CLInst, 256 GL::GLSLExtInst GLInst) const; 257 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I, const ExtInstList &ExtInsts) const; 259 260 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 261 MachineInstr &I) const; 262 263 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 264 MachineInstr &I) const; 265 266 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 267 MachineInstr &I, unsigned Opcode) const; 268 269 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 270 MachineInstr &I) const; 271 272 bool selectUnmergeValues(MachineInstr &I) const; 273 274 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 275 MachineInstr &I) const; 276 277 void selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 278 MachineInstr &I) const; 279 280 void selectImageWriteIntrinsic(MachineInstr &I) const; 281 282 // Utilities 283 std::pair<Register, bool> 284 buildI32Constant(uint32_t Val, MachineInstr &I, 285 const SPIRVType *ResType = nullptr) const; 286 287 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 288 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 289 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 290 MachineInstr &I) const; 291 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 292 293 bool wrapIntoSpecConstantOp(MachineInstr &I, 294 SmallVector<Register> &CompositeArgs) const; 295 296 Register getUcharPtrTypeReg(MachineInstr &I, 297 SPIRV::StorageClass::StorageClass SC) const; 298 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 299 Register Src, Register DestType, 300 uint32_t Opcode) const; 301 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 302 SPIRVType *SrcPtrTy) const; 303 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 304 uint32_t Binding, uint32_t ArraySize, 305 Register IndexReg, bool IsNonUniform, 306 MachineIRBuilder MIRBuilder) const; 307 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 308 void extractSubvector(Register &ResVReg, const SPIRVType *ResType, 309 Register &ReadReg, MachineInstr &InsertionPoint) const; 310 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 311 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 312 Register ResVReg, const SPIRVType *ResType, 313 MachineInstr &I) const; 314 }; 315 316 } // end anonymous namespace 317 318 #define GET_GLOBALISEL_IMPL 319 #include "SPIRVGenGlobalISel.inc" 320 #undef GET_GLOBALISEL_IMPL 321 322 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 323 const SPIRVSubtarget &ST, 324 const RegisterBankInfo &RBI) 325 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 326 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 327 #define GET_GLOBALISEL_PREDICATES_INIT 328 #include "SPIRVGenGlobalISel.inc" 329 #undef GET_GLOBALISEL_PREDICATES_INIT 330 #define GET_GLOBALISEL_TEMPORARIES_INIT 331 #include "SPIRVGenGlobalISel.inc" 332 #undef GET_GLOBALISEL_TEMPORARIES_INIT 333 { 334 } 335 336 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 337 CodeGenCoverage *CoverageInfo, 338 ProfileSummaryInfo *PSI, 339 BlockFrequencyInfo *BFI) { 340 MRI = &MF.getRegInfo(); 341 GR.setCurrentFunc(MF); 342 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 343 } 344 345 // Ensure that register classes correspond to pattern matching rules. 346 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 347 if (HasVRegsReset == &MF) 348 return; 349 HasVRegsReset = &MF; 350 351 MachineRegisterInfo &MRI = MF.getRegInfo(); 352 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 353 Register Reg = Register::index2VirtReg(I); 354 LLT RegType = MRI.getType(Reg); 355 if (RegType.isScalar()) 356 MRI.setType(Reg, LLT::scalar(64)); 357 else if (RegType.isPointer()) 358 MRI.setType(Reg, LLT::pointer(0, 64)); 359 else if (RegType.isVector()) 360 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 361 } 362 for (const auto &MBB : MF) { 363 for (const auto &MI : MBB) { 364 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 365 continue; 366 Register DstReg = MI.getOperand(0).getReg(); 367 LLT DstType = MRI.getType(DstReg); 368 Register SrcReg = MI.getOperand(1).getReg(); 369 LLT SrcType = MRI.getType(SrcReg); 370 if (DstType != SrcType) 371 MRI.setType(DstReg, MRI.getType(SrcReg)); 372 373 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 374 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 375 if (DstRC != SrcRC && SrcRC) 376 MRI.setRegClass(DstReg, SrcRC); 377 } 378 } 379 } 380 381 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 382 383 // Defined in SPIRVLegalizerInfo.cpp. 384 extern bool isTypeFoldingSupported(unsigned Opcode); 385 386 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 387 for (const auto &MO : MI.all_defs()) { 388 Register Reg = MO.getReg(); 389 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 390 return false; 391 } 392 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 393 MI.isLifetimeMarker()) 394 return false; 395 if (MI.isPHI()) 396 return true; 397 if (MI.mayStore() || MI.isCall() || 398 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 399 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 400 return false; 401 return true; 402 } 403 404 bool SPIRVInstructionSelector::select(MachineInstr &I) { 405 resetVRegsType(*I.getParent()->getParent()); 406 407 assert(I.getParent() && "Instruction should be in a basic block!"); 408 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 409 410 Register Opcode = I.getOpcode(); 411 // If it's not a GMIR instruction, we've selected it already. 412 if (!isPreISelGenericOpcode(Opcode)) { 413 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 414 Register DstReg = I.getOperand(0).getReg(); 415 Register SrcReg = I.getOperand(1).getReg(); 416 auto *Def = MRI->getVRegDef(SrcReg); 417 if (isTypeFoldingSupported(Def->getOpcode())) { 418 bool Res = selectImpl(I, *CoverageInfo); 419 LLVM_DEBUG({ 420 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 421 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 422 I.print(dbgs()); 423 } 424 }); 425 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 426 if (Res) { 427 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 428 DeadMIs.insert(Def); 429 return Res; 430 } 431 } 432 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 433 MRI->replaceRegWith(SrcReg, DstReg); 434 GR.invalidateMachineInstr(&I); 435 I.removeFromParent(); 436 return true; 437 } else if (I.getNumDefs() == 1) { 438 // Make all vregs 64 bits (for SPIR-V IDs). 439 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 440 } 441 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 442 } 443 444 if (DeadMIs.contains(&I)) { 445 // if the instruction has been already made dead by folding it away 446 // erase it 447 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 448 salvageDebugInfo(*MRI, I); 449 GR.invalidateMachineInstr(&I); 450 I.eraseFromParent(); 451 return true; 452 } 453 454 if (I.getNumOperands() != I.getNumExplicitOperands()) { 455 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 456 return false; 457 } 458 459 // Common code for getting return reg+type, and removing selected instr 460 // from parent occurs here. Instr-specific selection happens in spvSelect(). 461 bool HasDefs = I.getNumDefs() > 0; 462 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 463 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 464 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 465 if (spvSelect(ResVReg, ResType, I)) { 466 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 467 for (unsigned i = 0; i < I.getNumDefs(); ++i) 468 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 469 GR.invalidateMachineInstr(&I); 470 I.removeFromParent(); 471 return true; 472 } 473 return false; 474 } 475 476 static bool mayApplyGenericSelection(unsigned Opcode) { 477 switch (Opcode) { 478 case TargetOpcode::G_CONSTANT: 479 return false; 480 case TargetOpcode::G_SADDO: 481 case TargetOpcode::G_SSUBO: 482 return true; 483 } 484 return isTypeFoldingSupported(Opcode); 485 } 486 487 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 488 MachineInstr &I) const { 489 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 490 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 491 if (DstRC != SrcRC && SrcRC) 492 MRI->setRegClass(DestReg, SrcRC); 493 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 494 TII.get(TargetOpcode::COPY)) 495 .addDef(DestReg) 496 .addUse(SrcReg) 497 .constrainAllUses(TII, TRI, RBI); 498 } 499 500 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 501 const SPIRVType *ResType, 502 MachineInstr &I) const { 503 const unsigned Opcode = I.getOpcode(); 504 if (mayApplyGenericSelection(Opcode)) 505 return selectImpl(I, *CoverageInfo); 506 switch (Opcode) { 507 case TargetOpcode::G_CONSTANT: 508 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 509 I); 510 case TargetOpcode::G_GLOBAL_VALUE: 511 return selectGlobalValue(ResVReg, I); 512 case TargetOpcode::G_IMPLICIT_DEF: 513 return selectOpUndef(ResVReg, ResType, I); 514 case TargetOpcode::G_FREEZE: 515 return selectFreeze(ResVReg, ResType, I); 516 517 case TargetOpcode::G_INTRINSIC: 518 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 519 case TargetOpcode::G_INTRINSIC_CONVERGENT: 520 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 521 return selectIntrinsic(ResVReg, ResType, I); 522 case TargetOpcode::G_BITREVERSE: 523 return selectBitreverse(ResVReg, ResType, I); 524 525 case TargetOpcode::G_BUILD_VECTOR: 526 return selectBuildVector(ResVReg, ResType, I); 527 case TargetOpcode::G_SPLAT_VECTOR: 528 return selectSplatVector(ResVReg, ResType, I); 529 530 case TargetOpcode::G_SHUFFLE_VECTOR: { 531 MachineBasicBlock &BB = *I.getParent(); 532 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 533 .addDef(ResVReg) 534 .addUse(GR.getSPIRVTypeID(ResType)) 535 .addUse(I.getOperand(1).getReg()) 536 .addUse(I.getOperand(2).getReg()); 537 for (auto V : I.getOperand(3).getShuffleMask()) 538 MIB.addImm(V); 539 return MIB.constrainAllUses(TII, TRI, RBI); 540 } 541 case TargetOpcode::G_MEMMOVE: 542 case TargetOpcode::G_MEMCPY: 543 case TargetOpcode::G_MEMSET: 544 return selectMemOperation(ResVReg, I); 545 546 case TargetOpcode::G_ICMP: 547 return selectICmp(ResVReg, ResType, I); 548 case TargetOpcode::G_FCMP: 549 return selectFCmp(ResVReg, ResType, I); 550 551 case TargetOpcode::G_FRAME_INDEX: 552 return selectFrameIndex(ResVReg, ResType, I); 553 554 case TargetOpcode::G_LOAD: 555 return selectLoad(ResVReg, ResType, I); 556 case TargetOpcode::G_STORE: 557 return selectStore(I); 558 559 case TargetOpcode::G_BR: 560 return selectBranch(I); 561 case TargetOpcode::G_BRCOND: 562 return selectBranchCond(I); 563 564 case TargetOpcode::G_PHI: 565 return selectPhi(ResVReg, ResType, I); 566 567 case TargetOpcode::G_FPTOSI: 568 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 569 case TargetOpcode::G_FPTOUI: 570 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 571 572 case TargetOpcode::G_SITOFP: 573 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 574 case TargetOpcode::G_UITOFP: 575 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 576 577 case TargetOpcode::G_CTPOP: 578 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 579 case TargetOpcode::G_SMIN: 580 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 581 case TargetOpcode::G_UMIN: 582 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 583 584 case TargetOpcode::G_SMAX: 585 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 586 case TargetOpcode::G_UMAX: 587 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 588 589 case TargetOpcode::G_SCMP: 590 return selectSUCmp(ResVReg, ResType, I, true); 591 case TargetOpcode::G_UCMP: 592 return selectSUCmp(ResVReg, ResType, I, false); 593 594 case TargetOpcode::G_STRICT_FMA: 595 case TargetOpcode::G_FMA: 596 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 597 598 case TargetOpcode::G_STRICT_FLDEXP: 599 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 600 601 case TargetOpcode::G_FPOW: 602 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 603 case TargetOpcode::G_FPOWI: 604 return selectExtInst(ResVReg, ResType, I, CL::pown); 605 606 case TargetOpcode::G_FEXP: 607 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 608 case TargetOpcode::G_FEXP2: 609 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 610 611 case TargetOpcode::G_FLOG: 612 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 613 case TargetOpcode::G_FLOG2: 614 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 615 case TargetOpcode::G_FLOG10: 616 return selectLog10(ResVReg, ResType, I); 617 618 case TargetOpcode::G_FABS: 619 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 620 case TargetOpcode::G_ABS: 621 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 622 623 case TargetOpcode::G_FMINNUM: 624 case TargetOpcode::G_FMINIMUM: 625 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 626 case TargetOpcode::G_FMAXNUM: 627 case TargetOpcode::G_FMAXIMUM: 628 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 629 630 case TargetOpcode::G_FCOPYSIGN: 631 return selectExtInst(ResVReg, ResType, I, CL::copysign); 632 633 case TargetOpcode::G_FCEIL: 634 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 635 case TargetOpcode::G_FFLOOR: 636 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 637 638 case TargetOpcode::G_FCOS: 639 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 640 case TargetOpcode::G_FSIN: 641 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 642 case TargetOpcode::G_FTAN: 643 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 644 case TargetOpcode::G_FACOS: 645 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 646 case TargetOpcode::G_FASIN: 647 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 648 case TargetOpcode::G_FATAN: 649 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 650 case TargetOpcode::G_FATAN2: 651 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 652 case TargetOpcode::G_FCOSH: 653 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 654 case TargetOpcode::G_FSINH: 655 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 656 case TargetOpcode::G_FTANH: 657 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 658 659 case TargetOpcode::G_STRICT_FSQRT: 660 case TargetOpcode::G_FSQRT: 661 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 662 663 case TargetOpcode::G_CTTZ: 664 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 665 return selectExtInst(ResVReg, ResType, I, CL::ctz); 666 case TargetOpcode::G_CTLZ: 667 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 668 return selectExtInst(ResVReg, ResType, I, CL::clz); 669 670 case TargetOpcode::G_INTRINSIC_ROUND: 671 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 672 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 673 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 674 case TargetOpcode::G_INTRINSIC_TRUNC: 675 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 676 case TargetOpcode::G_FRINT: 677 case TargetOpcode::G_FNEARBYINT: 678 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 679 680 case TargetOpcode::G_SMULH: 681 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 682 case TargetOpcode::G_UMULH: 683 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 684 685 case TargetOpcode::G_SADDSAT: 686 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 687 case TargetOpcode::G_UADDSAT: 688 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 689 case TargetOpcode::G_SSUBSAT: 690 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 691 case TargetOpcode::G_USUBSAT: 692 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 693 694 case TargetOpcode::G_UADDO: 695 return selectOverflowArith(ResVReg, ResType, I, 696 ResType->getOpcode() == SPIRV::OpTypeVector 697 ? SPIRV::OpIAddCarryV 698 : SPIRV::OpIAddCarryS); 699 case TargetOpcode::G_USUBO: 700 return selectOverflowArith(ResVReg, ResType, I, 701 ResType->getOpcode() == SPIRV::OpTypeVector 702 ? SPIRV::OpISubBorrowV 703 : SPIRV::OpISubBorrowS); 704 case TargetOpcode::G_UMULO: 705 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 706 case TargetOpcode::G_SMULO: 707 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 708 709 case TargetOpcode::G_SEXT: 710 return selectExt(ResVReg, ResType, I, true); 711 case TargetOpcode::G_ANYEXT: 712 case TargetOpcode::G_ZEXT: 713 return selectExt(ResVReg, ResType, I, false); 714 case TargetOpcode::G_TRUNC: 715 return selectTrunc(ResVReg, ResType, I); 716 case TargetOpcode::G_FPTRUNC: 717 case TargetOpcode::G_FPEXT: 718 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 719 720 case TargetOpcode::G_PTRTOINT: 721 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 722 case TargetOpcode::G_INTTOPTR: 723 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 724 case TargetOpcode::G_BITCAST: 725 return selectBitcast(ResVReg, ResType, I); 726 case TargetOpcode::G_ADDRSPACE_CAST: 727 return selectAddrSpaceCast(ResVReg, ResType, I); 728 case TargetOpcode::G_PTR_ADD: { 729 // Currently, we get G_PTR_ADD only applied to global variables. 730 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 731 Register GV = I.getOperand(1).getReg(); 732 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 733 (void)II; 734 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 735 (*II).getOpcode() == TargetOpcode::COPY || 736 (*II).getOpcode() == SPIRV::OpVariable) && 737 isImm(I.getOperand(2), MRI)); 738 // It may be the initialization of a global variable. 739 bool IsGVInit = false; 740 for (MachineRegisterInfo::use_instr_iterator 741 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 742 UseEnd = MRI->use_instr_end(); 743 UseIt != UseEnd; UseIt = std::next(UseIt)) { 744 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 745 (*UseIt).getOpcode() == SPIRV::OpVariable) { 746 IsGVInit = true; 747 break; 748 } 749 } 750 MachineBasicBlock &BB = *I.getParent(); 751 if (!IsGVInit) { 752 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 753 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 754 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 755 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 756 // Build a new virtual register that is associated with the required 757 // data type. 758 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 759 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 760 // Having a correctly typed base we are ready to build the actually 761 // required GEP. It may not be a constant though, because all Operands 762 // of OpSpecConstantOp is to originate from other const instructions, 763 // and only the AccessChain named opcodes accept a global OpVariable 764 // instruction. We can't use an AccessChain opcode because of the type 765 // mismatch between result and base types. 766 if (!GR.isBitcastCompatible(ResType, GVType)) 767 report_fatal_error( 768 "incompatible result and operand types in a bitcast"); 769 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 770 MachineInstrBuilder MIB = 771 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 772 .addDef(NewVReg) 773 .addUse(ResTypeReg) 774 .addUse(GV); 775 return MIB.constrainAllUses(TII, TRI, RBI) && 776 BuildMI(BB, I, I.getDebugLoc(), 777 TII.get(STI.isVulkanEnv() 778 ? SPIRV::OpInBoundsAccessChain 779 : SPIRV::OpInBoundsPtrAccessChain)) 780 .addDef(ResVReg) 781 .addUse(ResTypeReg) 782 .addUse(NewVReg) 783 .addUse(I.getOperand(2).getReg()) 784 .constrainAllUses(TII, TRI, RBI); 785 } else { 786 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 787 .addDef(ResVReg) 788 .addUse(GR.getSPIRVTypeID(ResType)) 789 .addImm( 790 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 791 .addUse(GV) 792 .addUse(I.getOperand(2).getReg()) 793 .constrainAllUses(TII, TRI, RBI); 794 } 795 } 796 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 797 // initialize a global variable with a constant expression (e.g., the test 798 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 799 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 800 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 801 .addDef(ResVReg) 802 .addUse(GR.getSPIRVTypeID(ResType)) 803 .addImm(static_cast<uint32_t>( 804 SPIRV::Opcode::InBoundsPtrAccessChain)) 805 .addUse(GV) 806 .addUse(Idx) 807 .addUse(I.getOperand(2).getReg()); 808 return MIB.constrainAllUses(TII, TRI, RBI); 809 } 810 811 case TargetOpcode::G_ATOMICRMW_OR: 812 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 813 case TargetOpcode::G_ATOMICRMW_ADD: 814 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 815 case TargetOpcode::G_ATOMICRMW_AND: 816 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 817 case TargetOpcode::G_ATOMICRMW_MAX: 818 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 819 case TargetOpcode::G_ATOMICRMW_MIN: 820 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 821 case TargetOpcode::G_ATOMICRMW_SUB: 822 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 823 case TargetOpcode::G_ATOMICRMW_XOR: 824 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 825 case TargetOpcode::G_ATOMICRMW_UMAX: 826 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 827 case TargetOpcode::G_ATOMICRMW_UMIN: 828 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 829 case TargetOpcode::G_ATOMICRMW_XCHG: 830 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 831 case TargetOpcode::G_ATOMIC_CMPXCHG: 832 return selectAtomicCmpXchg(ResVReg, ResType, I); 833 834 case TargetOpcode::G_ATOMICRMW_FADD: 835 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 836 case TargetOpcode::G_ATOMICRMW_FSUB: 837 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 838 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 839 SPIRV::OpFNegate); 840 case TargetOpcode::G_ATOMICRMW_FMIN: 841 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 842 case TargetOpcode::G_ATOMICRMW_FMAX: 843 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 844 845 case TargetOpcode::G_FENCE: 846 return selectFence(I); 847 848 case TargetOpcode::G_STACKSAVE: 849 return selectStackSave(ResVReg, ResType, I); 850 case TargetOpcode::G_STACKRESTORE: 851 return selectStackRestore(I); 852 853 case TargetOpcode::G_UNMERGE_VALUES: 854 return selectUnmergeValues(I); 855 856 // Discard gen opcodes for intrinsics which we do not expect to actually 857 // represent code after lowering or intrinsics which are not implemented but 858 // should not crash when found in a customer's LLVM IR input. 859 case TargetOpcode::G_TRAP: 860 case TargetOpcode::G_DEBUGTRAP: 861 case TargetOpcode::G_UBSANTRAP: 862 case TargetOpcode::DBG_LABEL: 863 return true; 864 865 default: 866 return false; 867 } 868 } 869 870 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 871 const SPIRVType *ResType, 872 MachineInstr &I, 873 GL::GLSLExtInst GLInst) const { 874 return selectExtInst(ResVReg, ResType, I, 875 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 876 } 877 878 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 879 const SPIRVType *ResType, 880 MachineInstr &I, 881 CL::OpenCLExtInst CLInst) const { 882 return selectExtInst(ResVReg, ResType, I, 883 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 884 } 885 886 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 887 const SPIRVType *ResType, 888 MachineInstr &I, 889 CL::OpenCLExtInst CLInst, 890 GL::GLSLExtInst GLInst) const { 891 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 892 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 893 return selectExtInst(ResVReg, ResType, I, ExtInsts); 894 } 895 896 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 897 const SPIRVType *ResType, 898 MachineInstr &I, 899 const ExtInstList &Insts) const { 900 901 for (const auto &Ex : Insts) { 902 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 903 uint32_t Opcode = Ex.second; 904 if (STI.canUseExtInstSet(Set)) { 905 MachineBasicBlock &BB = *I.getParent(); 906 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 907 .addDef(ResVReg) 908 .addUse(GR.getSPIRVTypeID(ResType)) 909 .addImm(static_cast<uint32_t>(Set)) 910 .addImm(Opcode); 911 const unsigned NumOps = I.getNumOperands(); 912 unsigned Index = 1; 913 if (Index < NumOps && 914 I.getOperand(Index).getType() == 915 MachineOperand::MachineOperandType::MO_IntrinsicID) 916 Index = 2; 917 for (; Index < NumOps; ++Index) 918 MIB.add(I.getOperand(Index)); 919 return MIB.constrainAllUses(TII, TRI, RBI); 920 } 921 } 922 return false; 923 } 924 925 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 926 const SPIRVType *ResType, 927 MachineInstr &I, 928 std::vector<Register> Srcs, 929 unsigned Opcode) const { 930 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 931 .addDef(ResVReg) 932 .addUse(GR.getSPIRVTypeID(ResType)); 933 for (Register SReg : Srcs) { 934 MIB.addUse(SReg); 935 } 936 return MIB.constrainAllUses(TII, TRI, RBI); 937 } 938 939 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 940 const SPIRVType *ResType, 941 MachineInstr &I, 942 unsigned Opcode) const { 943 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 944 Register SrcReg = I.getOperand(1).getReg(); 945 bool IsGV = false; 946 for (MachineRegisterInfo::def_instr_iterator DefIt = 947 MRI->def_instr_begin(SrcReg); 948 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 949 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 950 IsGV = true; 951 break; 952 } 953 } 954 if (IsGV) { 955 uint32_t SpecOpcode = 0; 956 switch (Opcode) { 957 case SPIRV::OpConvertPtrToU: 958 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 959 break; 960 case SPIRV::OpConvertUToPtr: 961 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 962 break; 963 } 964 if (SpecOpcode) 965 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 966 TII.get(SPIRV::OpSpecConstantOp)) 967 .addDef(ResVReg) 968 .addUse(GR.getSPIRVTypeID(ResType)) 969 .addImm(SpecOpcode) 970 .addUse(SrcReg) 971 .constrainAllUses(TII, TRI, RBI); 972 } 973 } 974 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 975 Opcode); 976 } 977 978 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 979 const SPIRVType *ResType, 980 MachineInstr &I) const { 981 Register OpReg = I.getOperand(1).getReg(); 982 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 983 if (!GR.isBitcastCompatible(ResType, OpType)) 984 report_fatal_error("incompatible result and operand types in a bitcast"); 985 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 986 } 987 988 static void addMemoryOperands(MachineMemOperand *MemOp, 989 MachineInstrBuilder &MIB) { 990 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 991 if (MemOp->isVolatile()) 992 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 993 if (MemOp->isNonTemporal()) 994 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 995 if (MemOp->getAlign().value()) 996 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 997 998 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 999 MIB.addImm(SpvMemOp); 1000 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 1001 MIB.addImm(MemOp->getAlign().value()); 1002 } 1003 } 1004 1005 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1006 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1007 if (Flags & MachineMemOperand::Flags::MOVolatile) 1008 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1009 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1010 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1011 1012 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1013 MIB.addImm(SpvMemOp); 1014 } 1015 1016 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1017 const SPIRVType *ResType, 1018 MachineInstr &I) const { 1019 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1020 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1021 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1022 .addDef(ResVReg) 1023 .addUse(GR.getSPIRVTypeID(ResType)) 1024 .addUse(Ptr); 1025 if (!I.getNumMemOperands()) { 1026 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1027 I.getOpcode() == 1028 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1029 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1030 } else { 1031 addMemoryOperands(*I.memoperands_begin(), MIB); 1032 } 1033 return MIB.constrainAllUses(TII, TRI, RBI); 1034 } 1035 1036 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1037 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1038 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1039 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1040 MachineBasicBlock &BB = *I.getParent(); 1041 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1042 .addUse(Ptr) 1043 .addUse(StoreVal); 1044 if (!I.getNumMemOperands()) { 1045 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1046 I.getOpcode() == 1047 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1048 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1049 } else { 1050 addMemoryOperands(*I.memoperands_begin(), MIB); 1051 } 1052 return MIB.constrainAllUses(TII, TRI, RBI); 1053 } 1054 1055 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1056 const SPIRVType *ResType, 1057 MachineInstr &I) const { 1058 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1059 report_fatal_error( 1060 "llvm.stacksave intrinsic: this instruction requires the following " 1061 "SPIR-V extension: SPV_INTEL_variable_length_array", 1062 false); 1063 MachineBasicBlock &BB = *I.getParent(); 1064 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1065 .addDef(ResVReg) 1066 .addUse(GR.getSPIRVTypeID(ResType)) 1067 .constrainAllUses(TII, TRI, RBI); 1068 } 1069 1070 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1071 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1072 report_fatal_error( 1073 "llvm.stackrestore intrinsic: this instruction requires the following " 1074 "SPIR-V extension: SPV_INTEL_variable_length_array", 1075 false); 1076 if (!I.getOperand(0).isReg()) 1077 return false; 1078 MachineBasicBlock &BB = *I.getParent(); 1079 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1080 .addUse(I.getOperand(0).getReg()) 1081 .constrainAllUses(TII, TRI, RBI); 1082 } 1083 1084 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1085 MachineInstr &I) const { 1086 MachineBasicBlock &BB = *I.getParent(); 1087 Register SrcReg = I.getOperand(1).getReg(); 1088 bool Result = true; 1089 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1090 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1091 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1092 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1093 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1094 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1095 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1096 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1097 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1098 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1099 Function &CurFunction = GR.CurMF->getFunction(); 1100 Type *LLVMArrTy = 1101 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1102 // Module takes ownership of the global var. 1103 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1104 true, GlobalValue::InternalLinkage, 1105 Constant::getNullValue(LLVMArrTy)); 1106 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1107 GR.add(GV, GR.CurMF, VarReg); 1108 GR.addGlobalObject(GV, GR.CurMF, VarReg); 1109 1110 Result &= 1111 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1112 .addDef(VarReg) 1113 .addUse(GR.getSPIRVTypeID(VarTy)) 1114 .addImm(SPIRV::StorageClass::UniformConstant) 1115 .addUse(Const) 1116 .constrainAllUses(TII, TRI, RBI); 1117 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1118 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1119 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1120 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1121 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1122 } 1123 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1124 .addUse(I.getOperand(0).getReg()) 1125 .addUse(SrcReg) 1126 .addUse(I.getOperand(2).getReg()); 1127 if (I.getNumMemOperands()) 1128 addMemoryOperands(*I.memoperands_begin(), MIB); 1129 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1130 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1131 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1132 return Result; 1133 } 1134 1135 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1136 const SPIRVType *ResType, 1137 MachineInstr &I, 1138 unsigned NewOpcode, 1139 unsigned NegateOpcode) const { 1140 bool Result = true; 1141 assert(I.hasOneMemOperand()); 1142 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1143 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1144 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1145 auto ScopeConstant = buildI32Constant(Scope, I); 1146 Register ScopeReg = ScopeConstant.first; 1147 Result &= ScopeConstant.second; 1148 1149 Register Ptr = I.getOperand(1).getReg(); 1150 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1151 // auto ScSem = 1152 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1153 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1154 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1155 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1156 Register MemSemReg = MemSemConstant.first; 1157 Result &= MemSemConstant.second; 1158 1159 Register ValueReg = I.getOperand(2).getReg(); 1160 if (NegateOpcode != 0) { 1161 // Translation with negative value operand is requested 1162 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1163 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1164 ValueReg = TmpReg; 1165 } 1166 1167 return Result && 1168 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1169 .addDef(ResVReg) 1170 .addUse(GR.getSPIRVTypeID(ResType)) 1171 .addUse(Ptr) 1172 .addUse(ScopeReg) 1173 .addUse(MemSemReg) 1174 .addUse(ValueReg) 1175 .constrainAllUses(TII, TRI, RBI); 1176 } 1177 1178 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1179 unsigned ArgI = I.getNumOperands() - 1; 1180 Register SrcReg = 1181 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1182 SPIRVType *DefType = 1183 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1184 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1185 report_fatal_error( 1186 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1187 1188 SPIRVType *ScalarType = 1189 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1190 MachineBasicBlock &BB = *I.getParent(); 1191 bool Res = false; 1192 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1193 Register ResVReg = I.getOperand(i).getReg(); 1194 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1195 if (!ResType) { 1196 // There was no "assign type" actions, let's fix this now 1197 ResType = ScalarType; 1198 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1199 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1200 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1201 } 1202 auto MIB = 1203 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1204 .addDef(ResVReg) 1205 .addUse(GR.getSPIRVTypeID(ResType)) 1206 .addUse(SrcReg) 1207 .addImm(static_cast<int64_t>(i)); 1208 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1209 } 1210 return Res; 1211 } 1212 1213 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1214 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1215 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1216 auto MemSemConstant = buildI32Constant(MemSem, I); 1217 Register MemSemReg = MemSemConstant.first; 1218 bool Result = MemSemConstant.second; 1219 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1220 uint32_t Scope = static_cast<uint32_t>( 1221 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1222 auto ScopeConstant = buildI32Constant(Scope, I); 1223 Register ScopeReg = ScopeConstant.first; 1224 Result &= ScopeConstant.second; 1225 MachineBasicBlock &BB = *I.getParent(); 1226 return Result && 1227 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1228 .addUse(ScopeReg) 1229 .addUse(MemSemReg) 1230 .constrainAllUses(TII, TRI, RBI); 1231 } 1232 1233 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1234 const SPIRVType *ResType, 1235 MachineInstr &I, 1236 unsigned Opcode) const { 1237 Type *ResTy = nullptr; 1238 StringRef ResName; 1239 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1240 report_fatal_error( 1241 "Not enough info to select the arithmetic with overflow instruction"); 1242 if (!ResTy || !ResTy->isStructTy()) 1243 report_fatal_error("Expect struct type result for the arithmetic " 1244 "with overflow instruction"); 1245 // "Result Type must be from OpTypeStruct. The struct must have two members, 1246 // and the two members must be the same type." 1247 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1248 ResTy = StructType::get(ResElemTy, ResElemTy); 1249 // Build SPIR-V types and constant(s) if needed. 1250 MachineIRBuilder MIRBuilder(I); 1251 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1252 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1253 assert(I.getNumDefs() > 1 && "Not enought operands"); 1254 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1255 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1256 if (N > 1) 1257 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1258 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1259 Register ZeroReg = buildZerosVal(ResType, I); 1260 // A new virtual register to store the result struct. 1261 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1262 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1263 // Build the result name if needed. 1264 if (ResName.size() > 0) 1265 buildOpName(StructVReg, ResName, MIRBuilder); 1266 // Build the arithmetic with overflow instruction. 1267 MachineBasicBlock &BB = *I.getParent(); 1268 auto MIB = 1269 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1270 .addDef(StructVReg) 1271 .addUse(GR.getSPIRVTypeID(StructType)); 1272 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1273 MIB.addUse(I.getOperand(i).getReg()); 1274 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1275 // Build instructions to extract fields of the instruction's result. 1276 // A new virtual register to store the higher part of the result struct. 1277 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1278 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1279 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1280 auto MIB = 1281 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1282 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1283 .addUse(GR.getSPIRVTypeID(ResType)) 1284 .addUse(StructVReg) 1285 .addImm(i); 1286 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1287 } 1288 // Build boolean value from the higher part. 1289 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1290 .addDef(I.getOperand(1).getReg()) 1291 .addUse(BoolTypeReg) 1292 .addUse(HigherVReg) 1293 .addUse(ZeroReg) 1294 .constrainAllUses(TII, TRI, RBI); 1295 } 1296 1297 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1298 const SPIRVType *ResType, 1299 MachineInstr &I) const { 1300 bool Result = true; 1301 Register ScopeReg; 1302 Register MemSemEqReg; 1303 Register MemSemNeqReg; 1304 Register Ptr = I.getOperand(2).getReg(); 1305 if (!isa<GIntrinsic>(I)) { 1306 assert(I.hasOneMemOperand()); 1307 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1308 unsigned Scope = static_cast<uint32_t>(getMemScope( 1309 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1310 auto ScopeConstant = buildI32Constant(Scope, I); 1311 ScopeReg = ScopeConstant.first; 1312 Result &= ScopeConstant.second; 1313 1314 unsigned ScSem = static_cast<uint32_t>( 1315 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1316 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1317 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1318 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1319 MemSemEqReg = MemSemEqConstant.first; 1320 Result &= MemSemEqConstant.second; 1321 AtomicOrdering FO = MemOp->getFailureOrdering(); 1322 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1323 if (MemSemEq == MemSemNeq) 1324 MemSemNeqReg = MemSemEqReg; 1325 else { 1326 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1327 MemSemNeqReg = MemSemNeqConstant.first; 1328 Result &= MemSemNeqConstant.second; 1329 } 1330 } else { 1331 ScopeReg = I.getOperand(5).getReg(); 1332 MemSemEqReg = I.getOperand(6).getReg(); 1333 MemSemNeqReg = I.getOperand(7).getReg(); 1334 } 1335 1336 Register Cmp = I.getOperand(3).getReg(); 1337 Register Val = I.getOperand(4).getReg(); 1338 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1339 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1340 const DebugLoc &DL = I.getDebugLoc(); 1341 Result &= 1342 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1343 .addDef(ACmpRes) 1344 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1345 .addUse(Ptr) 1346 .addUse(ScopeReg) 1347 .addUse(MemSemEqReg) 1348 .addUse(MemSemNeqReg) 1349 .addUse(Val) 1350 .addUse(Cmp) 1351 .constrainAllUses(TII, TRI, RBI); 1352 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1353 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1354 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1355 .addDef(CmpSuccReg) 1356 .addUse(GR.getSPIRVTypeID(BoolTy)) 1357 .addUse(ACmpRes) 1358 .addUse(Cmp) 1359 .constrainAllUses(TII, TRI, RBI); 1360 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1361 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1362 .addDef(TmpReg) 1363 .addUse(GR.getSPIRVTypeID(ResType)) 1364 .addUse(ACmpRes) 1365 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1366 .addImm(0) 1367 .constrainAllUses(TII, TRI, RBI); 1368 return Result && 1369 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1370 .addDef(ResVReg) 1371 .addUse(GR.getSPIRVTypeID(ResType)) 1372 .addUse(CmpSuccReg) 1373 .addUse(TmpReg) 1374 .addImm(1) 1375 .constrainAllUses(TII, TRI, RBI); 1376 } 1377 1378 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1379 switch (SC) { 1380 case SPIRV::StorageClass::Workgroup: 1381 case SPIRV::StorageClass::CrossWorkgroup: 1382 case SPIRV::StorageClass::Function: 1383 return true; 1384 default: 1385 return false; 1386 } 1387 } 1388 1389 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1390 switch (SC) { 1391 case SPIRV::StorageClass::DeviceOnlyINTEL: 1392 case SPIRV::StorageClass::HostOnlyINTEL: 1393 return true; 1394 default: 1395 return false; 1396 } 1397 } 1398 1399 // Returns true ResVReg is referred only from global vars and OpName's. 1400 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1401 bool IsGRef = false; 1402 bool IsAllowedRefs = 1403 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1404 [&IsGRef](auto const &It) { 1405 unsigned Opcode = It.getOpcode(); 1406 if (Opcode == SPIRV::OpConstantComposite || 1407 Opcode == SPIRV::OpVariable || 1408 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1409 return IsGRef = true; 1410 return Opcode == SPIRV::OpName; 1411 }); 1412 return IsAllowedRefs && IsGRef; 1413 } 1414 1415 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1416 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1417 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1418 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1419 } 1420 1421 MachineInstrBuilder 1422 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1423 Register Src, Register DestType, 1424 uint32_t Opcode) const { 1425 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1426 TII.get(SPIRV::OpSpecConstantOp)) 1427 .addDef(Dest) 1428 .addUse(DestType) 1429 .addImm(Opcode) 1430 .addUse(Src); 1431 } 1432 1433 MachineInstrBuilder 1434 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1435 SPIRVType *SrcPtrTy) const { 1436 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1437 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1438 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1439 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1440 SPIRV::StorageClass::Generic), 1441 GR.getPointerSize())); 1442 MachineFunction *MF = I.getParent()->getParent(); 1443 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1444 MachineInstrBuilder MIB = buildSpecConstantOp( 1445 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1446 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1447 GR.add(MIB.getInstr(), MF, Tmp); 1448 return MIB; 1449 } 1450 1451 // In SPIR-V address space casting can only happen to and from the Generic 1452 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1453 // pointers to and from Generic pointers. As such, we can convert e.g. from 1454 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1455 // other combinations can only be done by a bitcast, and are probably not safe. 1456 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1457 const SPIRVType *ResType, 1458 MachineInstr &I) const { 1459 MachineBasicBlock &BB = *I.getParent(); 1460 const DebugLoc &DL = I.getDebugLoc(); 1461 1462 Register SrcPtr = I.getOperand(1).getReg(); 1463 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1464 1465 // don't generate a cast for a null that may be represented by OpTypeInt 1466 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1467 ResType->getOpcode() != SPIRV::OpTypePointer) 1468 return BuildCOPY(ResVReg, SrcPtr, I); 1469 1470 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1471 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1472 1473 if (isASCastInGVar(MRI, ResVReg)) { 1474 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1475 // are expressed by OpSpecConstantOp with an Opcode. 1476 // TODO: maybe insert a check whether the Kernel capability was declared and 1477 // so PtrCastToGeneric/GenericCastToPtr are available. 1478 unsigned SpecOpcode = 1479 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1480 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1481 : (SrcSC == SPIRV::StorageClass::Generic && 1482 isGenericCastablePtr(DstSC) 1483 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1484 : 0); 1485 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1486 // correct value of ResType and use general i8* instead. Maybe this should 1487 // be addressed in the emit-intrinsic step to infer a correct 1488 // OpConstantComposite type. 1489 if (SpecOpcode) { 1490 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1491 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1492 .constrainAllUses(TII, TRI, RBI); 1493 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1494 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1495 return MIB.constrainAllUses(TII, TRI, RBI) && 1496 buildSpecConstantOp( 1497 I, ResVReg, MIB->getOperand(0).getReg(), 1498 getUcharPtrTypeReg(I, DstSC), 1499 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1500 .constrainAllUses(TII, TRI, RBI); 1501 } 1502 } 1503 1504 // don't generate a cast between identical storage classes 1505 if (SrcSC == DstSC) 1506 return BuildCOPY(ResVReg, SrcPtr, I); 1507 1508 if ((SrcSC == SPIRV::StorageClass::Function && 1509 DstSC == SPIRV::StorageClass::Private) || 1510 (DstSC == SPIRV::StorageClass::Function && 1511 SrcSC == SPIRV::StorageClass::Private)) 1512 return BuildCOPY(ResVReg, SrcPtr, I); 1513 1514 // Casting from an eligible pointer to Generic. 1515 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1516 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1517 // Casting from Generic to an eligible pointer. 1518 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1519 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1520 // Casting between 2 eligible pointers using Generic as an intermediary. 1521 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1522 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1523 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1524 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1525 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1526 .addDef(Tmp) 1527 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1528 .addUse(SrcPtr) 1529 .constrainAllUses(TII, TRI, RBI); 1530 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1531 .addDef(ResVReg) 1532 .addUse(GR.getSPIRVTypeID(ResType)) 1533 .addUse(Tmp) 1534 .constrainAllUses(TII, TRI, RBI); 1535 } 1536 1537 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1538 // be applied 1539 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1540 return selectUnOp(ResVReg, ResType, I, 1541 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1542 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1543 return selectUnOp(ResVReg, ResType, I, 1544 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1545 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1546 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1547 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1548 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1549 1550 // Bitcast for pointers requires that the address spaces must match 1551 return false; 1552 } 1553 1554 static unsigned getFCmpOpcode(unsigned PredNum) { 1555 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1556 switch (Pred) { 1557 case CmpInst::FCMP_OEQ: 1558 return SPIRV::OpFOrdEqual; 1559 case CmpInst::FCMP_OGE: 1560 return SPIRV::OpFOrdGreaterThanEqual; 1561 case CmpInst::FCMP_OGT: 1562 return SPIRV::OpFOrdGreaterThan; 1563 case CmpInst::FCMP_OLE: 1564 return SPIRV::OpFOrdLessThanEqual; 1565 case CmpInst::FCMP_OLT: 1566 return SPIRV::OpFOrdLessThan; 1567 case CmpInst::FCMP_ONE: 1568 return SPIRV::OpFOrdNotEqual; 1569 case CmpInst::FCMP_ORD: 1570 return SPIRV::OpOrdered; 1571 case CmpInst::FCMP_UEQ: 1572 return SPIRV::OpFUnordEqual; 1573 case CmpInst::FCMP_UGE: 1574 return SPIRV::OpFUnordGreaterThanEqual; 1575 case CmpInst::FCMP_UGT: 1576 return SPIRV::OpFUnordGreaterThan; 1577 case CmpInst::FCMP_ULE: 1578 return SPIRV::OpFUnordLessThanEqual; 1579 case CmpInst::FCMP_ULT: 1580 return SPIRV::OpFUnordLessThan; 1581 case CmpInst::FCMP_UNE: 1582 return SPIRV::OpFUnordNotEqual; 1583 case CmpInst::FCMP_UNO: 1584 return SPIRV::OpUnordered; 1585 default: 1586 llvm_unreachable("Unknown predicate type for FCmp"); 1587 } 1588 } 1589 1590 static unsigned getICmpOpcode(unsigned PredNum) { 1591 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1592 switch (Pred) { 1593 case CmpInst::ICMP_EQ: 1594 return SPIRV::OpIEqual; 1595 case CmpInst::ICMP_NE: 1596 return SPIRV::OpINotEqual; 1597 case CmpInst::ICMP_SGE: 1598 return SPIRV::OpSGreaterThanEqual; 1599 case CmpInst::ICMP_SGT: 1600 return SPIRV::OpSGreaterThan; 1601 case CmpInst::ICMP_SLE: 1602 return SPIRV::OpSLessThanEqual; 1603 case CmpInst::ICMP_SLT: 1604 return SPIRV::OpSLessThan; 1605 case CmpInst::ICMP_UGE: 1606 return SPIRV::OpUGreaterThanEqual; 1607 case CmpInst::ICMP_UGT: 1608 return SPIRV::OpUGreaterThan; 1609 case CmpInst::ICMP_ULE: 1610 return SPIRV::OpULessThanEqual; 1611 case CmpInst::ICMP_ULT: 1612 return SPIRV::OpULessThan; 1613 default: 1614 llvm_unreachable("Unknown predicate type for ICmp"); 1615 } 1616 } 1617 1618 static unsigned getPtrCmpOpcode(unsigned Pred) { 1619 switch (static_cast<CmpInst::Predicate>(Pred)) { 1620 case CmpInst::ICMP_EQ: 1621 return SPIRV::OpPtrEqual; 1622 case CmpInst::ICMP_NE: 1623 return SPIRV::OpPtrNotEqual; 1624 default: 1625 llvm_unreachable("Unknown predicate type for pointer comparison"); 1626 } 1627 } 1628 1629 // Return the logical operation, or abort if none exists. 1630 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1631 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1632 switch (Pred) { 1633 case CmpInst::ICMP_EQ: 1634 return SPIRV::OpLogicalEqual; 1635 case CmpInst::ICMP_NE: 1636 return SPIRV::OpLogicalNotEqual; 1637 default: 1638 llvm_unreachable("Unknown predicate type for Bool comparison"); 1639 } 1640 } 1641 1642 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1643 if (!LLVMFloatTy) 1644 return APFloat::getZero(APFloat::IEEEsingle()); 1645 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1646 case Type::HalfTyID: 1647 return APFloat::getZero(APFloat::IEEEhalf()); 1648 default: 1649 case Type::FloatTyID: 1650 return APFloat::getZero(APFloat::IEEEsingle()); 1651 case Type::DoubleTyID: 1652 return APFloat::getZero(APFloat::IEEEdouble()); 1653 } 1654 } 1655 1656 static APFloat getOneFP(const Type *LLVMFloatTy) { 1657 if (!LLVMFloatTy) 1658 return APFloat::getOne(APFloat::IEEEsingle()); 1659 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1660 case Type::HalfTyID: 1661 return APFloat::getOne(APFloat::IEEEhalf()); 1662 default: 1663 case Type::FloatTyID: 1664 return APFloat::getOne(APFloat::IEEEsingle()); 1665 case Type::DoubleTyID: 1666 return APFloat::getOne(APFloat::IEEEdouble()); 1667 } 1668 } 1669 1670 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1671 const SPIRVType *ResType, 1672 MachineInstr &I, 1673 unsigned OpAnyOrAll) const { 1674 assert(I.getNumOperands() == 3); 1675 assert(I.getOperand(2).isReg()); 1676 MachineBasicBlock &BB = *I.getParent(); 1677 Register InputRegister = I.getOperand(2).getReg(); 1678 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1679 1680 if (!InputType) 1681 report_fatal_error("Input Type could not be determined."); 1682 1683 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1684 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1685 if (IsBoolTy && !IsVectorTy) { 1686 assert(ResVReg == I.getOperand(0).getReg()); 1687 return BuildCOPY(ResVReg, InputRegister, I); 1688 } 1689 1690 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1691 unsigned SpirvNotEqualId = 1692 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1693 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1694 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1695 Register NotEqualReg = ResVReg; 1696 1697 if (IsVectorTy) { 1698 NotEqualReg = IsBoolTy ? InputRegister 1699 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1700 const unsigned NumElts = InputType->getOperand(2).getImm(); 1701 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1702 } 1703 1704 bool Result = true; 1705 if (!IsBoolTy) { 1706 Register ConstZeroReg = 1707 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1708 1709 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1710 .addDef(NotEqualReg) 1711 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1712 .addUse(InputRegister) 1713 .addUse(ConstZeroReg) 1714 .constrainAllUses(TII, TRI, RBI); 1715 } 1716 1717 if (!IsVectorTy) 1718 return Result; 1719 1720 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1721 .addDef(ResVReg) 1722 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1723 .addUse(NotEqualReg) 1724 .constrainAllUses(TII, TRI, RBI); 1725 } 1726 1727 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1728 const SPIRVType *ResType, 1729 MachineInstr &I) const { 1730 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1731 } 1732 1733 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1734 const SPIRVType *ResType, 1735 MachineInstr &I) const { 1736 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1737 } 1738 1739 // Select the OpDot instruction for the given float dot 1740 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1741 const SPIRVType *ResType, 1742 MachineInstr &I) const { 1743 assert(I.getNumOperands() == 4); 1744 assert(I.getOperand(2).isReg()); 1745 assert(I.getOperand(3).isReg()); 1746 1747 [[maybe_unused]] SPIRVType *VecType = 1748 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1749 1750 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1751 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1752 "dot product requires a vector of at least 2 components"); 1753 1754 [[maybe_unused]] SPIRVType *EltType = 1755 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1756 1757 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1758 1759 MachineBasicBlock &BB = *I.getParent(); 1760 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1761 .addDef(ResVReg) 1762 .addUse(GR.getSPIRVTypeID(ResType)) 1763 .addUse(I.getOperand(2).getReg()) 1764 .addUse(I.getOperand(3).getReg()) 1765 .constrainAllUses(TII, TRI, RBI); 1766 } 1767 1768 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1769 const SPIRVType *ResType, 1770 MachineInstr &I, 1771 bool Signed) const { 1772 assert(I.getNumOperands() == 4); 1773 assert(I.getOperand(2).isReg()); 1774 assert(I.getOperand(3).isReg()); 1775 MachineBasicBlock &BB = *I.getParent(); 1776 1777 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1778 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1779 .addDef(ResVReg) 1780 .addUse(GR.getSPIRVTypeID(ResType)) 1781 .addUse(I.getOperand(2).getReg()) 1782 .addUse(I.getOperand(3).getReg()) 1783 .constrainAllUses(TII, TRI, RBI); 1784 } 1785 1786 // Since pre-1.6 SPIRV has no integer dot implementation, 1787 // expand by piecewise multiplying and adding the results 1788 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1789 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1790 assert(I.getNumOperands() == 4); 1791 assert(I.getOperand(2).isReg()); 1792 assert(I.getOperand(3).isReg()); 1793 MachineBasicBlock &BB = *I.getParent(); 1794 1795 // Multiply the vectors, then sum the results 1796 Register Vec0 = I.getOperand(2).getReg(); 1797 Register Vec1 = I.getOperand(3).getReg(); 1798 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1799 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1800 1801 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1802 .addDef(TmpVec) 1803 .addUse(GR.getSPIRVTypeID(VecType)) 1804 .addUse(Vec0) 1805 .addUse(Vec1) 1806 .constrainAllUses(TII, TRI, RBI); 1807 1808 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1809 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1810 "dot product requires a vector of at least 2 components"); 1811 1812 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1813 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1814 .addDef(Res) 1815 .addUse(GR.getSPIRVTypeID(ResType)) 1816 .addUse(TmpVec) 1817 .addImm(0) 1818 .constrainAllUses(TII, TRI, RBI); 1819 1820 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1821 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1822 1823 Result &= 1824 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1825 .addDef(Elt) 1826 .addUse(GR.getSPIRVTypeID(ResType)) 1827 .addUse(TmpVec) 1828 .addImm(i) 1829 .constrainAllUses(TII, TRI, RBI); 1830 1831 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1832 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1833 : ResVReg; 1834 1835 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1836 .addDef(Sum) 1837 .addUse(GR.getSPIRVTypeID(ResType)) 1838 .addUse(Res) 1839 .addUse(Elt) 1840 .constrainAllUses(TII, TRI, RBI); 1841 Res = Sum; 1842 } 1843 1844 return Result; 1845 } 1846 1847 template <bool Signed> 1848 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1849 const SPIRVType *ResType, 1850 MachineInstr &I) const { 1851 assert(I.getNumOperands() == 5); 1852 assert(I.getOperand(2).isReg()); 1853 assert(I.getOperand(3).isReg()); 1854 assert(I.getOperand(4).isReg()); 1855 MachineBasicBlock &BB = *I.getParent(); 1856 1857 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1858 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1859 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1860 .addDef(Dot) 1861 .addUse(GR.getSPIRVTypeID(ResType)) 1862 .addUse(I.getOperand(2).getReg()) 1863 .addUse(I.getOperand(3).getReg()) 1864 .constrainAllUses(TII, TRI, RBI); 1865 1866 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1867 .addDef(ResVReg) 1868 .addUse(GR.getSPIRVTypeID(ResType)) 1869 .addUse(Dot) 1870 .addUse(I.getOperand(4).getReg()) 1871 .constrainAllUses(TII, TRI, RBI); 1872 } 1873 1874 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1875 // extract the elements of the packed inputs, multiply them and add the result 1876 // to the accumulator. 1877 template <bool Signed> 1878 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1879 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1880 assert(I.getNumOperands() == 5); 1881 assert(I.getOperand(2).isReg()); 1882 assert(I.getOperand(3).isReg()); 1883 assert(I.getOperand(4).isReg()); 1884 MachineBasicBlock &BB = *I.getParent(); 1885 1886 bool Result = true; 1887 1888 // Acc = C 1889 Register Acc = I.getOperand(4).getReg(); 1890 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1891 auto ExtractOp = 1892 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1893 1894 // Extract the i8 element, multiply and add it to the accumulator 1895 for (unsigned i = 0; i < 4; i++) { 1896 // A[i] 1897 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1898 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1899 .addDef(AElt) 1900 .addUse(GR.getSPIRVTypeID(ResType)) 1901 .addUse(I.getOperand(2).getReg()) 1902 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1903 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1904 .constrainAllUses(TII, TRI, RBI); 1905 1906 // B[i] 1907 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1908 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1909 .addDef(BElt) 1910 .addUse(GR.getSPIRVTypeID(ResType)) 1911 .addUse(I.getOperand(3).getReg()) 1912 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1913 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1914 .constrainAllUses(TII, TRI, RBI); 1915 1916 // A[i] * B[i] 1917 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1918 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1919 .addDef(Mul) 1920 .addUse(GR.getSPIRVTypeID(ResType)) 1921 .addUse(AElt) 1922 .addUse(BElt) 1923 .constrainAllUses(TII, TRI, RBI); 1924 1925 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1926 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1927 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1928 .addDef(MaskMul) 1929 .addUse(GR.getSPIRVTypeID(ResType)) 1930 .addUse(Mul) 1931 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1932 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1933 .constrainAllUses(TII, TRI, RBI); 1934 1935 // Acc = Acc + A[i] * B[i] 1936 Register Sum = 1937 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1938 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1939 .addDef(Sum) 1940 .addUse(GR.getSPIRVTypeID(ResType)) 1941 .addUse(Acc) 1942 .addUse(MaskMul) 1943 .constrainAllUses(TII, TRI, RBI); 1944 1945 Acc = Sum; 1946 } 1947 1948 return Result; 1949 } 1950 1951 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1952 /// does not have a saturate builtin. 1953 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1954 const SPIRVType *ResType, 1955 MachineInstr &I) const { 1956 assert(I.getNumOperands() == 3); 1957 assert(I.getOperand(2).isReg()); 1958 MachineBasicBlock &BB = *I.getParent(); 1959 Register VZero = buildZerosValF(ResType, I); 1960 Register VOne = buildOnesValF(ResType, I); 1961 1962 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1963 .addDef(ResVReg) 1964 .addUse(GR.getSPIRVTypeID(ResType)) 1965 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1966 .addImm(GL::FClamp) 1967 .addUse(I.getOperand(2).getReg()) 1968 .addUse(VZero) 1969 .addUse(VOne) 1970 .constrainAllUses(TII, TRI, RBI); 1971 } 1972 1973 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1974 const SPIRVType *ResType, 1975 MachineInstr &I) const { 1976 assert(I.getNumOperands() == 3); 1977 assert(I.getOperand(2).isReg()); 1978 MachineBasicBlock &BB = *I.getParent(); 1979 Register InputRegister = I.getOperand(2).getReg(); 1980 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1981 auto &DL = I.getDebugLoc(); 1982 1983 if (!InputType) 1984 report_fatal_error("Input Type could not be determined."); 1985 1986 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1987 1988 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1989 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1990 1991 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1992 1993 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1994 Register SignReg = NeedsConversion 1995 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1996 : ResVReg; 1997 1998 bool Result = 1999 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 2000 .addDef(SignReg) 2001 .addUse(GR.getSPIRVTypeID(InputType)) 2002 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2003 .addImm(SignOpcode) 2004 .addUse(InputRegister) 2005 .constrainAllUses(TII, TRI, RBI); 2006 2007 if (NeedsConversion) { 2008 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2009 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2010 .addDef(ResVReg) 2011 .addUse(GR.getSPIRVTypeID(ResType)) 2012 .addUse(SignReg) 2013 .constrainAllUses(TII, TRI, RBI); 2014 } 2015 2016 return Result; 2017 } 2018 2019 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2020 const SPIRVType *ResType, 2021 MachineInstr &I, 2022 unsigned Opcode) const { 2023 MachineBasicBlock &BB = *I.getParent(); 2024 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2025 2026 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2027 .addDef(ResVReg) 2028 .addUse(GR.getSPIRVTypeID(ResType)) 2029 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2030 IntTy, TII)); 2031 2032 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2033 BMI.addUse(I.getOperand(J).getReg()); 2034 } 2035 2036 return BMI.constrainAllUses(TII, TRI, RBI); 2037 } 2038 2039 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2040 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2041 2042 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2043 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2044 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2045 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2046 SPIRV::OpGroupNonUniformBallot); 2047 2048 MachineBasicBlock &BB = *I.getParent(); 2049 Result &= 2050 BuildMI(BB, I, I.getDebugLoc(), 2051 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2052 .addDef(ResVReg) 2053 .addUse(GR.getSPIRVTypeID(ResType)) 2054 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2055 .addImm(SPIRV::GroupOperation::Reduce) 2056 .addUse(BallotReg) 2057 .constrainAllUses(TII, TRI, RBI); 2058 2059 return Result; 2060 } 2061 2062 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2063 const SPIRVType *ResType, 2064 MachineInstr &I) const { 2065 MachineBasicBlock &BB = *I.getParent(); 2066 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2067 .addDef(ResVReg) 2068 .addUse(GR.getSPIRVTypeID(ResType)) 2069 .addUse(I.getOperand(1).getReg()) 2070 .constrainAllUses(TII, TRI, RBI); 2071 } 2072 2073 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2074 const SPIRVType *ResType, 2075 MachineInstr &I) const { 2076 // There is no way to implement `freeze` correctly without support on SPIR-V 2077 // standard side, but we may at least address a simple (static) case when 2078 // undef/poison value presence is obvious. The main benefit of even 2079 // incomplete `freeze` support is preventing of translation from crashing due 2080 // to lack of support on legalization and instruction selection steps. 2081 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2082 return false; 2083 Register OpReg = I.getOperand(1).getReg(); 2084 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2085 Register Reg; 2086 switch (Def->getOpcode()) { 2087 case SPIRV::ASSIGN_TYPE: 2088 if (MachineInstr *AssignToDef = 2089 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2090 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2091 Reg = Def->getOperand(2).getReg(); 2092 } 2093 break; 2094 case SPIRV::OpUndef: 2095 Reg = Def->getOperand(1).getReg(); 2096 break; 2097 } 2098 unsigned DestOpCode; 2099 if (Reg.isValid()) { 2100 DestOpCode = SPIRV::OpConstantNull; 2101 } else { 2102 DestOpCode = TargetOpcode::COPY; 2103 Reg = OpReg; 2104 } 2105 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2106 .addDef(I.getOperand(0).getReg()) 2107 .addUse(Reg) 2108 .constrainAllUses(TII, TRI, RBI); 2109 } 2110 return false; 2111 } 2112 2113 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2114 const SPIRVType *ResType) { 2115 Register OpReg = ResType->getOperand(2).getReg(); 2116 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2117 if (!OpDef) 2118 return 0; 2119 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2120 OpDef->getOperand(1).isReg()) { 2121 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2122 OpDef = RefDef; 2123 } 2124 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2125 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2126 : 0; 2127 return N; 2128 } 2129 2130 // Return true if the type represents a constant register 2131 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2132 SmallPtrSet<SPIRVType *, 4> &Visited) { 2133 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2134 OpDef->getOperand(1).isReg()) { 2135 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2136 OpDef = RefDef; 2137 } 2138 2139 if (Visited.contains(OpDef)) 2140 return true; 2141 Visited.insert(OpDef); 2142 2143 unsigned Opcode = OpDef->getOpcode(); 2144 switch (Opcode) { 2145 case TargetOpcode::G_CONSTANT: 2146 case TargetOpcode::G_FCONSTANT: 2147 return true; 2148 case TargetOpcode::G_INTRINSIC: 2149 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2150 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2151 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2152 Intrinsic::spv_const_composite; 2153 case TargetOpcode::G_BUILD_VECTOR: 2154 case TargetOpcode::G_SPLAT_VECTOR: { 2155 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2156 i++) { 2157 SPIRVType *OpNestedDef = 2158 OpDef->getOperand(i).isReg() 2159 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2160 : nullptr; 2161 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2162 return false; 2163 } 2164 return true; 2165 } 2166 } 2167 return false; 2168 } 2169 2170 // Return true if the virtual register represents a constant 2171 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2172 SmallPtrSet<SPIRVType *, 4> Visited; 2173 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2174 return isConstReg(MRI, OpDef, Visited); 2175 return false; 2176 } 2177 2178 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2179 const SPIRVType *ResType, 2180 MachineInstr &I) const { 2181 unsigned N = 0; 2182 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2183 N = GR.getScalarOrVectorComponentCount(ResType); 2184 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2185 N = getArrayComponentCount(MRI, ResType); 2186 else 2187 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2188 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2189 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2190 2191 // check if we may construct a constant vector 2192 bool IsConst = true; 2193 for (unsigned i = I.getNumExplicitDefs(); 2194 i < I.getNumExplicitOperands() && IsConst; ++i) 2195 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2196 IsConst = false; 2197 2198 if (!IsConst && N < 2) 2199 report_fatal_error( 2200 "There must be at least two constituent operands in a vector"); 2201 2202 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2203 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2204 TII.get(IsConst ? SPIRV::OpConstantComposite 2205 : SPIRV::OpCompositeConstruct)) 2206 .addDef(ResVReg) 2207 .addUse(GR.getSPIRVTypeID(ResType)); 2208 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2209 MIB.addUse(I.getOperand(i).getReg()); 2210 return MIB.constrainAllUses(TII, TRI, RBI); 2211 } 2212 2213 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2214 const SPIRVType *ResType, 2215 MachineInstr &I) const { 2216 unsigned N = 0; 2217 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2218 N = GR.getScalarOrVectorComponentCount(ResType); 2219 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2220 N = getArrayComponentCount(MRI, ResType); 2221 else 2222 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2223 2224 unsigned OpIdx = I.getNumExplicitDefs(); 2225 if (!I.getOperand(OpIdx).isReg()) 2226 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2227 2228 // check if we may construct a constant vector 2229 Register OpReg = I.getOperand(OpIdx).getReg(); 2230 bool IsConst = isConstReg(MRI, OpReg); 2231 2232 if (!IsConst && N < 2) 2233 report_fatal_error( 2234 "There must be at least two constituent operands in a vector"); 2235 2236 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2237 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2238 TII.get(IsConst ? SPIRV::OpConstantComposite 2239 : SPIRV::OpCompositeConstruct)) 2240 .addDef(ResVReg) 2241 .addUse(GR.getSPIRVTypeID(ResType)); 2242 for (unsigned i = 0; i < N; ++i) 2243 MIB.addUse(OpReg); 2244 return MIB.constrainAllUses(TII, TRI, RBI); 2245 } 2246 2247 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2248 const SPIRVType *ResType, 2249 MachineInstr &I) const { 2250 2251 unsigned Opcode; 2252 2253 if (STI.canUseExtension( 2254 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2255 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2256 Opcode = SPIRV::OpDemoteToHelperInvocation; 2257 } else { 2258 Opcode = SPIRV::OpKill; 2259 // OpKill must be the last operation of any basic block. 2260 if (MachineInstr *NextI = I.getNextNode()) { 2261 GR.invalidateMachineInstr(NextI); 2262 NextI->removeFromParent(); 2263 } 2264 } 2265 2266 MachineBasicBlock &BB = *I.getParent(); 2267 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2268 .constrainAllUses(TII, TRI, RBI); 2269 } 2270 2271 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2272 const SPIRVType *ResType, 2273 unsigned CmpOpc, 2274 MachineInstr &I) const { 2275 Register Cmp0 = I.getOperand(2).getReg(); 2276 Register Cmp1 = I.getOperand(3).getReg(); 2277 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2278 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2279 "CMP operands should have the same type"); 2280 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2281 .addDef(ResVReg) 2282 .addUse(GR.getSPIRVTypeID(ResType)) 2283 .addUse(Cmp0) 2284 .addUse(Cmp1) 2285 .constrainAllUses(TII, TRI, RBI); 2286 } 2287 2288 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2289 const SPIRVType *ResType, 2290 MachineInstr &I) const { 2291 auto Pred = I.getOperand(1).getPredicate(); 2292 unsigned CmpOpc; 2293 2294 Register CmpOperand = I.getOperand(2).getReg(); 2295 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2296 CmpOpc = getPtrCmpOpcode(Pred); 2297 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2298 CmpOpc = getBoolCmpOpcode(Pred); 2299 else 2300 CmpOpc = getICmpOpcode(Pred); 2301 return selectCmp(ResVReg, ResType, CmpOpc, I); 2302 } 2303 2304 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2305 const MachineInstr &I, 2306 int OpIdx) const { 2307 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2308 "Expected G_FCONSTANT"); 2309 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2310 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2311 } 2312 2313 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2314 const MachineInstr &I, 2315 int OpIdx) const { 2316 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2317 "Expected G_CONSTANT"); 2318 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2319 } 2320 2321 std::pair<Register, bool> 2322 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2323 const SPIRVType *ResType) const { 2324 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2325 const SPIRVType *SpvI32Ty = 2326 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2327 // Find a constant in DT or build a new one. 2328 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2329 Register NewReg = GR.find(ConstInt, GR.CurMF); 2330 bool Result = true; 2331 if (!NewReg.isValid()) { 2332 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2333 GR.add(ConstInt, GR.CurMF, NewReg); 2334 MachineInstr *MI; 2335 MachineBasicBlock &BB = *I.getParent(); 2336 if (Val == 0) { 2337 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2338 .addDef(NewReg) 2339 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2340 } else { 2341 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2342 .addDef(NewReg) 2343 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2344 .addImm(APInt(32, Val).getZExtValue()); 2345 } 2346 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2347 } 2348 return {NewReg, Result}; 2349 } 2350 2351 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2352 const SPIRVType *ResType, 2353 MachineInstr &I) const { 2354 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2355 return selectCmp(ResVReg, ResType, CmpOp, I); 2356 } 2357 2358 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2359 MachineInstr &I) const { 2360 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2361 bool ZeroAsNull = STI.isOpenCLEnv(); 2362 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2363 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2364 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2365 } 2366 2367 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2368 MachineInstr &I) const { 2369 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2370 bool ZeroAsNull = STI.isOpenCLEnv(); 2371 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2372 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2373 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2374 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2375 } 2376 2377 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2378 MachineInstr &I) const { 2379 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2380 bool ZeroAsNull = STI.isOpenCLEnv(); 2381 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2382 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2383 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2384 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2385 } 2386 2387 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2388 const SPIRVType *ResType, 2389 MachineInstr &I) const { 2390 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2391 APInt One = 2392 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2393 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2394 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2395 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2396 } 2397 2398 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2399 const SPIRVType *ResType, 2400 MachineInstr &I, 2401 bool IsSigned) const { 2402 // To extend a bool, we need to use OpSelect between constants. 2403 Register ZeroReg = buildZerosVal(ResType, I); 2404 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2405 bool IsScalarBool = 2406 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2407 unsigned Opcode = 2408 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2409 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2410 .addDef(ResVReg) 2411 .addUse(GR.getSPIRVTypeID(ResType)) 2412 .addUse(I.getOperand(1).getReg()) 2413 .addUse(OneReg) 2414 .addUse(ZeroReg) 2415 .constrainAllUses(TII, TRI, RBI); 2416 } 2417 2418 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2419 const SPIRVType *ResType, 2420 MachineInstr &I, bool IsSigned, 2421 unsigned Opcode) const { 2422 Register SrcReg = I.getOperand(1).getReg(); 2423 // We can convert bool value directly to float type without OpConvert*ToF, 2424 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2425 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2426 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2427 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2428 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2429 const unsigned NumElts = ResType->getOperand(2).getImm(); 2430 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2431 } 2432 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2433 selectSelect(SrcReg, TmpType, I, false); 2434 } 2435 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2436 } 2437 2438 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2439 const SPIRVType *ResType, 2440 MachineInstr &I, bool IsSigned) const { 2441 Register SrcReg = I.getOperand(1).getReg(); 2442 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2443 return selectSelect(ResVReg, ResType, I, IsSigned); 2444 2445 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2446 if (SrcType == ResType) 2447 return BuildCOPY(ResVReg, SrcReg, I); 2448 2449 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2450 return selectUnOp(ResVReg, ResType, I, Opcode); 2451 } 2452 2453 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2454 const SPIRVType *ResType, 2455 MachineInstr &I, 2456 bool IsSigned) const { 2457 MachineIRBuilder MIRBuilder(I); 2458 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2459 MachineBasicBlock &BB = *I.getParent(); 2460 // Ensure we have bool. 2461 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2462 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2463 if (N > 1) 2464 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2465 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2466 // Build less-than-equal and less-than. 2467 // TODO: replace with one-liner createVirtualRegister() from 2468 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2469 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2470 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2471 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2472 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2473 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2474 : SPIRV::OpULessThanEqual)) 2475 .addDef(IsLessEqReg) 2476 .addUse(BoolTypeReg) 2477 .addUse(I.getOperand(1).getReg()) 2478 .addUse(I.getOperand(2).getReg()) 2479 .constrainAllUses(TII, TRI, RBI); 2480 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2481 MRI->setType(IsLessReg, LLT::scalar(64)); 2482 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2483 Result &= BuildMI(BB, I, I.getDebugLoc(), 2484 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2485 .addDef(IsLessReg) 2486 .addUse(BoolTypeReg) 2487 .addUse(I.getOperand(1).getReg()) 2488 .addUse(I.getOperand(2).getReg()) 2489 .constrainAllUses(TII, TRI, RBI); 2490 // Build selects. 2491 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2492 Register NegOneOrZeroReg = 2493 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2494 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2495 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2496 unsigned SelectOpcode = 2497 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2498 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2499 .addDef(NegOneOrZeroReg) 2500 .addUse(ResTypeReg) 2501 .addUse(IsLessReg) 2502 .addUse(buildOnesVal(true, ResType, I)) // -1 2503 .addUse(buildZerosVal(ResType, I)) 2504 .constrainAllUses(TII, TRI, RBI); 2505 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2506 .addDef(ResVReg) 2507 .addUse(ResTypeReg) 2508 .addUse(IsLessEqReg) 2509 .addUse(NegOneOrZeroReg) // -1 or 0 2510 .addUse(buildOnesVal(false, ResType, I)) 2511 .constrainAllUses(TII, TRI, RBI); 2512 } 2513 2514 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2515 Register ResVReg, 2516 MachineInstr &I, 2517 const SPIRVType *IntTy, 2518 const SPIRVType *BoolTy) const { 2519 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2520 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2521 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2522 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2523 Register Zero = buildZerosVal(IntTy, I); 2524 Register One = buildOnesVal(false, IntTy, I); 2525 MachineBasicBlock &BB = *I.getParent(); 2526 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2527 .addDef(BitIntReg) 2528 .addUse(GR.getSPIRVTypeID(IntTy)) 2529 .addUse(IntReg) 2530 .addUse(One) 2531 .constrainAllUses(TII, TRI, RBI); 2532 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2533 .addDef(ResVReg) 2534 .addUse(GR.getSPIRVTypeID(BoolTy)) 2535 .addUse(BitIntReg) 2536 .addUse(Zero) 2537 .constrainAllUses(TII, TRI, RBI); 2538 } 2539 2540 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2541 const SPIRVType *ResType, 2542 MachineInstr &I) const { 2543 Register IntReg = I.getOperand(1).getReg(); 2544 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2545 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2546 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2547 if (ArgType == ResType) 2548 return BuildCOPY(ResVReg, IntReg, I); 2549 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2550 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2551 return selectUnOp(ResVReg, ResType, I, Opcode); 2552 } 2553 2554 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2555 const SPIRVType *ResType, 2556 const APInt &Imm, 2557 MachineInstr &I) const { 2558 unsigned TyOpcode = ResType->getOpcode(); 2559 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2560 MachineBasicBlock &BB = *I.getParent(); 2561 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2562 Imm.isZero()) 2563 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2564 .addDef(ResVReg) 2565 .addUse(GR.getSPIRVTypeID(ResType)) 2566 .constrainAllUses(TII, TRI, RBI); 2567 if (TyOpcode == SPIRV::OpTypeInt) { 2568 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2569 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2570 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2571 } 2572 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2573 .addDef(ResVReg) 2574 .addUse(GR.getSPIRVTypeID(ResType)); 2575 // <=32-bit integers should be caught by the sdag pattern. 2576 assert(Imm.getBitWidth() > 32); 2577 addNumImm(Imm, MIB); 2578 return MIB.constrainAllUses(TII, TRI, RBI); 2579 } 2580 2581 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2582 const SPIRVType *ResType, 2583 MachineInstr &I) const { 2584 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2585 .addDef(ResVReg) 2586 .addUse(GR.getSPIRVTypeID(ResType)) 2587 .constrainAllUses(TII, TRI, RBI); 2588 } 2589 2590 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2591 assert(MO.isReg()); 2592 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2593 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2594 assert(TypeInst->getOperand(1).isReg()); 2595 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2596 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2597 } 2598 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2599 } 2600 2601 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2602 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2603 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2604 return TypeInst->getOperand(2).getImm(); 2605 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2606 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2607 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2608 } 2609 2610 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2611 const SPIRVType *ResType, 2612 MachineInstr &I) const { 2613 MachineBasicBlock &BB = *I.getParent(); 2614 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2615 .addDef(ResVReg) 2616 .addUse(GR.getSPIRVTypeID(ResType)) 2617 // object to insert 2618 .addUse(I.getOperand(3).getReg()) 2619 // composite to insert into 2620 .addUse(I.getOperand(2).getReg()); 2621 for (unsigned i = 4; i < I.getNumOperands(); i++) 2622 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2623 return MIB.constrainAllUses(TII, TRI, RBI); 2624 } 2625 2626 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2627 const SPIRVType *ResType, 2628 MachineInstr &I) const { 2629 MachineBasicBlock &BB = *I.getParent(); 2630 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2631 .addDef(ResVReg) 2632 .addUse(GR.getSPIRVTypeID(ResType)) 2633 .addUse(I.getOperand(2).getReg()); 2634 for (unsigned i = 3; i < I.getNumOperands(); i++) 2635 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2636 return MIB.constrainAllUses(TII, TRI, RBI); 2637 } 2638 2639 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2640 const SPIRVType *ResType, 2641 MachineInstr &I) const { 2642 if (isImm(I.getOperand(4), MRI)) 2643 return selectInsertVal(ResVReg, ResType, I); 2644 MachineBasicBlock &BB = *I.getParent(); 2645 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2646 .addDef(ResVReg) 2647 .addUse(GR.getSPIRVTypeID(ResType)) 2648 .addUse(I.getOperand(2).getReg()) 2649 .addUse(I.getOperand(3).getReg()) 2650 .addUse(I.getOperand(4).getReg()) 2651 .constrainAllUses(TII, TRI, RBI); 2652 } 2653 2654 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2655 const SPIRVType *ResType, 2656 MachineInstr &I) const { 2657 if (isImm(I.getOperand(3), MRI)) 2658 return selectExtractVal(ResVReg, ResType, I); 2659 MachineBasicBlock &BB = *I.getParent(); 2660 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2661 .addDef(ResVReg) 2662 .addUse(GR.getSPIRVTypeID(ResType)) 2663 .addUse(I.getOperand(2).getReg()) 2664 .addUse(I.getOperand(3).getReg()) 2665 .constrainAllUses(TII, TRI, RBI); 2666 } 2667 2668 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2669 const SPIRVType *ResType, 2670 MachineInstr &I) const { 2671 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2672 2673 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2674 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2675 // we have to use Op[InBounds]AccessChain. 2676 const unsigned Opcode = STI.isVulkanEnv() 2677 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2678 : SPIRV::OpAccessChain) 2679 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2680 : SPIRV::OpPtrAccessChain); 2681 2682 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2683 .addDef(ResVReg) 2684 .addUse(GR.getSPIRVTypeID(ResType)) 2685 // Object to get a pointer to. 2686 .addUse(I.getOperand(3).getReg()); 2687 // Adding indices. 2688 const unsigned StartingIndex = 2689 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2690 ? 5 2691 : 4; 2692 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2693 Res.addUse(I.getOperand(i).getReg()); 2694 return Res.constrainAllUses(TII, TRI, RBI); 2695 } 2696 2697 // Maybe wrap a value into OpSpecConstantOp 2698 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2699 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2700 bool Result = true; 2701 unsigned Lim = I.getNumExplicitOperands(); 2702 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2703 Register OpReg = I.getOperand(i).getReg(); 2704 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2705 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2706 SmallPtrSet<SPIRVType *, 4> Visited; 2707 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2708 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2709 GR.isAggregateType(OpType)) { 2710 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2711 // by selectAddrSpaceCast() 2712 CompositeArgs.push_back(OpReg); 2713 continue; 2714 } 2715 MachineFunction *MF = I.getMF(); 2716 Register WrapReg = GR.find(OpDefine, MF); 2717 if (WrapReg.isValid()) { 2718 CompositeArgs.push_back(WrapReg); 2719 continue; 2720 } 2721 // Create a new register for the wrapper 2722 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2723 GR.add(OpDefine, MF, WrapReg); 2724 CompositeArgs.push_back(WrapReg); 2725 // Decorate the wrapper register and generate a new instruction 2726 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2727 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2728 MachineBasicBlock &BB = *I.getParent(); 2729 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2730 .addDef(WrapReg) 2731 .addUse(GR.getSPIRVTypeID(OpType)) 2732 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2733 .addUse(OpReg) 2734 .constrainAllUses(TII, TRI, RBI); 2735 if (!Result) 2736 break; 2737 } 2738 return Result; 2739 } 2740 2741 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2742 const SPIRVType *ResType, 2743 MachineInstr &I) const { 2744 MachineBasicBlock &BB = *I.getParent(); 2745 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2746 switch (IID) { 2747 case Intrinsic::spv_load: 2748 return selectLoad(ResVReg, ResType, I); 2749 case Intrinsic::spv_store: 2750 return selectStore(I); 2751 case Intrinsic::spv_extractv: 2752 return selectExtractVal(ResVReg, ResType, I); 2753 case Intrinsic::spv_insertv: 2754 return selectInsertVal(ResVReg, ResType, I); 2755 case Intrinsic::spv_extractelt: 2756 return selectExtractElt(ResVReg, ResType, I); 2757 case Intrinsic::spv_insertelt: 2758 return selectInsertElt(ResVReg, ResType, I); 2759 case Intrinsic::spv_gep: 2760 return selectGEP(ResVReg, ResType, I); 2761 case Intrinsic::spv_unref_global: 2762 case Intrinsic::spv_init_global: { 2763 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2764 MachineInstr *Init = I.getNumExplicitOperands() > 2 2765 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2766 : nullptr; 2767 assert(MI); 2768 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2769 } 2770 case Intrinsic::spv_undef: { 2771 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2772 .addDef(ResVReg) 2773 .addUse(GR.getSPIRVTypeID(ResType)); 2774 return MIB.constrainAllUses(TII, TRI, RBI); 2775 } 2776 case Intrinsic::spv_const_composite: { 2777 // If no values are attached, the composite is null constant. 2778 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2779 // Select a proper instruction. 2780 unsigned Opcode = SPIRV::OpConstantNull; 2781 SmallVector<Register> CompositeArgs; 2782 if (!IsNull) { 2783 Opcode = SPIRV::OpConstantComposite; 2784 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2785 return false; 2786 } 2787 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2788 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2789 .addDef(ResVReg) 2790 .addUse(GR.getSPIRVTypeID(ResType)); 2791 // skip type MD node we already used when generated assign.type for this 2792 if (!IsNull) { 2793 for (Register OpReg : CompositeArgs) 2794 MIB.addUse(OpReg); 2795 } 2796 return MIB.constrainAllUses(TII, TRI, RBI); 2797 } 2798 case Intrinsic::spv_assign_name: { 2799 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2800 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2801 for (unsigned i = I.getNumExplicitDefs() + 2; 2802 i < I.getNumExplicitOperands(); ++i) { 2803 MIB.addImm(I.getOperand(i).getImm()); 2804 } 2805 return MIB.constrainAllUses(TII, TRI, RBI); 2806 } 2807 case Intrinsic::spv_switch: { 2808 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2809 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2810 if (I.getOperand(i).isReg()) 2811 MIB.addReg(I.getOperand(i).getReg()); 2812 else if (I.getOperand(i).isCImm()) 2813 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2814 else if (I.getOperand(i).isMBB()) 2815 MIB.addMBB(I.getOperand(i).getMBB()); 2816 else 2817 llvm_unreachable("Unexpected OpSwitch operand"); 2818 } 2819 return MIB.constrainAllUses(TII, TRI, RBI); 2820 } 2821 case Intrinsic::spv_loop_merge: 2822 case Intrinsic::spv_selection_merge: { 2823 const auto Opcode = IID == Intrinsic::spv_selection_merge 2824 ? SPIRV::OpSelectionMerge 2825 : SPIRV::OpLoopMerge; 2826 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2827 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2828 assert(I.getOperand(i).isMBB()); 2829 MIB.addMBB(I.getOperand(i).getMBB()); 2830 } 2831 MIB.addImm(SPIRV::SelectionControl::None); 2832 return MIB.constrainAllUses(TII, TRI, RBI); 2833 } 2834 case Intrinsic::spv_cmpxchg: 2835 return selectAtomicCmpXchg(ResVReg, ResType, I); 2836 case Intrinsic::spv_unreachable: 2837 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2838 .constrainAllUses(TII, TRI, RBI); 2839 case Intrinsic::spv_alloca: 2840 return selectFrameIndex(ResVReg, ResType, I); 2841 case Intrinsic::spv_alloca_array: 2842 return selectAllocaArray(ResVReg, ResType, I); 2843 case Intrinsic::spv_assume: 2844 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2845 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2846 .addUse(I.getOperand(1).getReg()) 2847 .constrainAllUses(TII, TRI, RBI); 2848 break; 2849 case Intrinsic::spv_expect: 2850 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2851 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2852 .addDef(ResVReg) 2853 .addUse(GR.getSPIRVTypeID(ResType)) 2854 .addUse(I.getOperand(2).getReg()) 2855 .addUse(I.getOperand(3).getReg()) 2856 .constrainAllUses(TII, TRI, RBI); 2857 break; 2858 case Intrinsic::arithmetic_fence: 2859 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2860 return BuildMI(BB, I, I.getDebugLoc(), 2861 TII.get(SPIRV::OpArithmeticFenceEXT)) 2862 .addDef(ResVReg) 2863 .addUse(GR.getSPIRVTypeID(ResType)) 2864 .addUse(I.getOperand(2).getReg()) 2865 .constrainAllUses(TII, TRI, RBI); 2866 else 2867 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2868 break; 2869 case Intrinsic::spv_thread_id: 2870 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 2871 // intrinsic in LLVM IR for SPIR-V backend. 2872 // 2873 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 2874 // `GlobalInvocationId` builtin variable 2875 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 2876 ResType, I); 2877 case Intrinsic::spv_thread_id_in_group: 2878 // The HLSL SV_GroupThreadId semantic is lowered to 2879 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 2880 // 2881 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 2882 // translated to a `LocalInvocationId` builtin variable 2883 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 2884 ResType, I); 2885 case Intrinsic::spv_group_id: 2886 // The HLSL SV_GroupId semantic is lowered to 2887 // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. 2888 // 2889 // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` 2890 // builtin variable 2891 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, 2892 I); 2893 case Intrinsic::spv_fdot: 2894 return selectFloatDot(ResVReg, ResType, I); 2895 case Intrinsic::spv_udot: 2896 case Intrinsic::spv_sdot: 2897 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2898 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2899 return selectIntegerDot(ResVReg, ResType, I, 2900 /*Signed=*/IID == Intrinsic::spv_sdot); 2901 return selectIntegerDotExpansion(ResVReg, ResType, I); 2902 case Intrinsic::spv_dot4add_i8packed: 2903 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2904 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2905 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2906 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2907 case Intrinsic::spv_dot4add_u8packed: 2908 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2909 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2910 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2911 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2912 case Intrinsic::spv_all: 2913 return selectAll(ResVReg, ResType, I); 2914 case Intrinsic::spv_any: 2915 return selectAny(ResVReg, ResType, I); 2916 case Intrinsic::spv_cross: 2917 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2918 case Intrinsic::spv_distance: 2919 return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); 2920 case Intrinsic::spv_lerp: 2921 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2922 case Intrinsic::spv_length: 2923 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2924 case Intrinsic::spv_degrees: 2925 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2926 case Intrinsic::spv_frac: 2927 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2928 case Intrinsic::spv_normalize: 2929 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2930 case Intrinsic::spv_rsqrt: 2931 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2932 case Intrinsic::spv_sign: 2933 return selectSign(ResVReg, ResType, I); 2934 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2935 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2936 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2937 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2938 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2939 bool Result = true; 2940 auto MemSemConstant = 2941 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2942 Register MemSemReg = MemSemConstant.first; 2943 Result &= MemSemConstant.second; 2944 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2945 Register ScopeReg = ScopeConstant.first; 2946 Result &= ScopeConstant.second; 2947 MachineBasicBlock &BB = *I.getParent(); 2948 return Result && 2949 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2950 .addUse(ScopeReg) 2951 .addUse(ScopeReg) 2952 .addUse(MemSemReg) 2953 .constrainAllUses(TII, TRI, RBI); 2954 } 2955 case Intrinsic::spv_lifetime_start: 2956 case Intrinsic::spv_lifetime_end: { 2957 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2958 : SPIRV::OpLifetimeStop; 2959 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2960 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2961 if (Size == -1) 2962 Size = 0; 2963 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2964 .addUse(PtrReg) 2965 .addImm(Size) 2966 .constrainAllUses(TII, TRI, RBI); 2967 } 2968 case Intrinsic::spv_saturate: 2969 return selectSaturate(ResVReg, ResType, I); 2970 case Intrinsic::spv_nclamp: 2971 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2972 case Intrinsic::spv_uclamp: 2973 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2974 case Intrinsic::spv_sclamp: 2975 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2976 case Intrinsic::spv_wave_active_countbits: 2977 return selectWaveActiveCountBits(ResVReg, ResType, I); 2978 case Intrinsic::spv_wave_all: 2979 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 2980 case Intrinsic::spv_wave_any: 2981 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 2982 case Intrinsic::spv_wave_is_first_lane: 2983 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 2984 case Intrinsic::spv_wave_readlane: 2985 return selectWaveOpInst(ResVReg, ResType, I, 2986 SPIRV::OpGroupNonUniformShuffle); 2987 case Intrinsic::spv_step: 2988 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2989 case Intrinsic::spv_radians: 2990 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2991 // Discard intrinsics which we do not expect to actually represent code after 2992 // lowering or intrinsics which are not implemented but should not crash when 2993 // found in a customer's LLVM IR input. 2994 case Intrinsic::instrprof_increment: 2995 case Intrinsic::instrprof_increment_step: 2996 case Intrinsic::instrprof_value_profile: 2997 break; 2998 // Discard internal intrinsics. 2999 case Intrinsic::spv_value_md: 3000 break; 3001 case Intrinsic::spv_resource_handlefrombinding: { 3002 return selectHandleFromBinding(ResVReg, ResType, I); 3003 } 3004 case Intrinsic::spv_resource_store_typedbuffer: { 3005 selectImageWriteIntrinsic(I); 3006 return true; 3007 } 3008 case Intrinsic::spv_resource_load_typedbuffer: { 3009 selectReadImageIntrinsic(ResVReg, ResType, I); 3010 return true; 3011 } 3012 case Intrinsic::spv_discard: { 3013 return selectDiscard(ResVReg, ResType, I); 3014 } 3015 default: { 3016 std::string DiagMsg; 3017 raw_string_ostream OS(DiagMsg); 3018 I.print(OS); 3019 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3020 report_fatal_error(DiagMsg.c_str(), false); 3021 } 3022 } 3023 return true; 3024 } 3025 3026 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3027 const SPIRVType *ResType, 3028 MachineInstr &I) const { 3029 3030 uint32_t Set = foldImm(I.getOperand(2), MRI); 3031 uint32_t Binding = foldImm(I.getOperand(3), MRI); 3032 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 3033 Register IndexReg = I.getOperand(5).getReg(); 3034 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 3035 3036 MachineIRBuilder MIRBuilder(I); 3037 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3038 IndexReg, IsNonUniform, MIRBuilder); 3039 3040 if (IsNonUniform) 3041 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 3042 3043 // TODO: For now we assume the resource is an image, which needs to be 3044 // loaded to get the handle. That will not be true for storage buffers. 3045 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3046 .addDef(ResVReg) 3047 .addUse(GR.getSPIRVTypeID(ResType)) 3048 .addUse(VarReg) 3049 .constrainAllUses(TII, TRI, RBI); 3050 } 3051 3052 void SPIRVInstructionSelector::selectReadImageIntrinsic( 3053 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3054 3055 // If the load of the image is in a different basic block, then 3056 // this will generate invalid code. A proper solution is to move 3057 // the OpLoad from selectHandleFromBinding here. However, to do 3058 // that we will need to change the return type of the intrinsic. 3059 // We will do that when we can, but for now trying to move forward with other 3060 // issues. 3061 Register ImageReg = I.getOperand(2).getReg(); 3062 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3063 "The image must be loaded in the same basic block as its use."); 3064 3065 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3066 if (ResultSize == 4) { 3067 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3068 .addDef(ResVReg) 3069 .addUse(GR.getSPIRVTypeID(ResType)) 3070 .addUse(ImageReg) 3071 .addUse(I.getOperand(3).getReg()); 3072 return; 3073 } 3074 3075 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 3076 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3077 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3078 .addDef(ReadReg) 3079 .addUse(GR.getSPIRVTypeID(ReadType)) 3080 .addUse(ImageReg) 3081 .addUse(I.getOperand(3).getReg()); 3082 3083 if (ResultSize == 1) { 3084 BuildMI(*I.getParent(), I, I.getDebugLoc(), 3085 TII.get(SPIRV::OpCompositeExtract)) 3086 .addDef(ResVReg) 3087 .addUse(GR.getSPIRVTypeID(ResType)) 3088 .addUse(ReadReg) 3089 .addImm(0); 3090 return; 3091 } 3092 extractSubvector(ResVReg, ResType, ReadReg, I); 3093 } 3094 3095 void SPIRVInstructionSelector::extractSubvector( 3096 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3097 MachineInstr &InsertionPoint) const { 3098 SPIRVType *InputType = GR.getResultType(ReadReg); 3099 [[maybe_unused]] uint64_t InputSize = 3100 GR.getScalarOrVectorComponentCount(InputType); 3101 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3102 assert(InputSize > 1 && "The input must be a vector."); 3103 assert(ResultSize > 1 && "The result must be a vector."); 3104 assert(ResultSize < InputSize && 3105 "Cannot extract more element than there are in the input."); 3106 SmallVector<Register> ComponentRegisters; 3107 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3108 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3109 for (uint64_t I = 0; I < ResultSize; I++) { 3110 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3111 BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3112 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3113 .addDef(ComponentReg) 3114 .addUse(ScalarType->getOperand(0).getReg()) 3115 .addUse(ReadReg) 3116 .addImm(I); 3117 ComponentRegisters.emplace_back(ComponentReg); 3118 } 3119 3120 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3121 InsertionPoint.getDebugLoc(), 3122 TII.get(SPIRV::OpCompositeConstruct)) 3123 .addDef(ResVReg) 3124 .addUse(GR.getSPIRVTypeID(ResType)); 3125 3126 for (Register ComponentReg : ComponentRegisters) 3127 MIB.addUse(ComponentReg); 3128 } 3129 3130 void SPIRVInstructionSelector::selectImageWriteIntrinsic( 3131 MachineInstr &I) const { 3132 // If the load of the image is in a different basic block, then 3133 // this will generate invalid code. A proper solution is to move 3134 // the OpLoad from selectHandleFromBinding here. However, to do 3135 // that we will need to change the return type of the intrinsic. 3136 // We will do that when we can, but for now trying to move forward with other 3137 // issues. 3138 Register ImageReg = I.getOperand(1).getReg(); 3139 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3140 "The image must be loaded in the same basic block as its use."); 3141 Register CoordinateReg = I.getOperand(2).getReg(); 3142 Register DataReg = I.getOperand(3).getReg(); 3143 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3144 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3145 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite)) 3146 .addUse(ImageReg) 3147 .addUse(CoordinateReg) 3148 .addUse(DataReg); 3149 } 3150 3151 Register SPIRVInstructionSelector::buildPointerToResource( 3152 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3153 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3154 MachineIRBuilder MIRBuilder) const { 3155 if (ArraySize == 1) 3156 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3157 MIRBuilder); 3158 3159 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3160 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3161 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3162 VarType, Set, Binding, MIRBuilder); 3163 3164 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3165 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3166 3167 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3168 if (IsNonUniform) { 3169 // It is unclear which value needs to be marked an non-uniform, so both 3170 // the index and the access changed are decorated as non-uniform. 3171 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3172 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3173 } 3174 3175 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3176 .addDef(AcReg) 3177 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3178 .addUse(VarReg) 3179 .addUse(IndexReg); 3180 3181 return AcReg; 3182 } 3183 3184 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3185 const SPIRVType *ResType, 3186 MachineInstr &I, 3187 bool IsSigned) const { 3188 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3189 // zero or sign extend 3190 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3191 bool Result = 3192 selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, Opcode); 3193 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3194 } 3195 3196 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3197 const SPIRVType *ResType, 3198 MachineInstr &I, 3199 Register SrcReg, 3200 bool IsSigned) const { 3201 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3202 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3203 .addDef(ResVReg) 3204 .addUse(GR.getSPIRVTypeID(ResType)) 3205 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3206 .addImm(Opcode) 3207 .addUse(SrcReg) 3208 .constrainAllUses(TII, TRI, RBI); 3209 } 3210 3211 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3212 const SPIRVType *ResType, 3213 MachineInstr &I, 3214 bool IsSigned) const { 3215 Register OpReg = I.getOperand(2).getReg(); 3216 // 1. split our int64 into 2 pieces using a bitcast 3217 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3218 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3219 MachineIRBuilder MIRBuilder(I); 3220 SPIRVType *postCastT = 3221 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3222 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3223 bool Result = 3224 selectOpWithSrcs(bitcastReg, postCastT, I, {OpReg}, SPIRV::OpBitcast); 3225 3226 // 2. call firstbithigh 3227 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3228 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3229 3230 // 3. split result vector into high bits and low bits 3231 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3232 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3233 3234 bool ZeroAsNull = STI.isOpenCLEnv(); 3235 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3236 if (isScalarRes) { 3237 // if scalar do a vector extract 3238 Result &= selectOpWithSrcs( 3239 HighReg, ResType, I, 3240 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3241 SPIRV::OpVectorExtractDynamic); 3242 Result &= selectOpWithSrcs( 3243 LowReg, ResType, I, 3244 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3245 SPIRV::OpVectorExtractDynamic); 3246 } else { // vector case do a shufflevector 3247 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3248 TII.get(SPIRV::OpVectorShuffle)) 3249 .addDef(HighReg) 3250 .addUse(GR.getSPIRVTypeID(ResType)) 3251 .addUse(FBHReg) 3252 .addUse(FBHReg); 3253 // ^^ this vector will not be selected from; could be empty 3254 unsigned j; 3255 for (j = 0; j < count * 2; j += 2) { 3256 MIB.addImm(j); 3257 } 3258 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3259 3260 // get low bits 3261 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3262 TII.get(SPIRV::OpVectorShuffle)) 3263 .addDef(LowReg) 3264 .addUse(GR.getSPIRVTypeID(ResType)) 3265 .addUse(FBHReg) 3266 .addUse(FBHReg); 3267 // ^^ this vector will not be selected from; could be empty 3268 for (j = 1; j < count * 2; j += 2) { 3269 MIB.addImm(j); 3270 } 3271 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3272 } 3273 3274 // 4. check if result of each top 32 bits is == -1 3275 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3276 Register NegOneReg; 3277 Register Reg0; 3278 Register Reg32; 3279 unsigned selectOp; 3280 unsigned addOp; 3281 if (isScalarRes) { 3282 NegOneReg = 3283 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3284 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3285 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3286 selectOp = SPIRV::OpSelectSISCond; 3287 addOp = SPIRV::OpIAddS; 3288 } else { 3289 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3290 NegOneReg = 3291 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3292 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3293 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3294 selectOp = SPIRV::OpSelectVIVCond; 3295 addOp = SPIRV::OpIAddV; 3296 } 3297 3298 // check if the high bits are == -1; true if -1 3299 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3300 Result &= selectOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3301 SPIRV::OpIEqual); 3302 3303 // Select low bits if true in BReg, otherwise high bits 3304 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3305 Result &= 3306 selectOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, selectOp); 3307 3308 // Add 32 for high bits, 0 for low bits 3309 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3310 Result &= selectOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3311 3312 return Result && 3313 selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3314 } 3315 3316 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3317 const SPIRVType *ResType, 3318 MachineInstr &I, 3319 bool IsSigned) const { 3320 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3321 Register OpReg = I.getOperand(2).getReg(); 3322 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3323 3324 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3325 case 16: 3326 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3327 case 32: 3328 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3329 case 64: 3330 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3331 default: 3332 report_fatal_error( 3333 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3334 } 3335 } 3336 3337 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3338 const SPIRVType *ResType, 3339 MachineInstr &I) const { 3340 // there was an allocation size parameter to the allocation instruction 3341 // that is not 1 3342 MachineBasicBlock &BB = *I.getParent(); 3343 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3344 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3345 .addDef(ResVReg) 3346 .addUse(GR.getSPIRVTypeID(ResType)) 3347 .addUse(I.getOperand(2).getReg()) 3348 .constrainAllUses(TII, TRI, RBI); 3349 if (!STI.isVulkanEnv()) { 3350 unsigned Alignment = I.getOperand(3).getImm(); 3351 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3352 } 3353 return Res; 3354 } 3355 3356 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3357 const SPIRVType *ResType, 3358 MachineInstr &I) const { 3359 // Change order of instructions if needed: all OpVariable instructions in a 3360 // function must be the first instructions in the first block 3361 auto It = getOpVariableMBBIt(I); 3362 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3363 TII.get(SPIRV::OpVariable)) 3364 .addDef(ResVReg) 3365 .addUse(GR.getSPIRVTypeID(ResType)) 3366 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3367 .constrainAllUses(TII, TRI, RBI); 3368 if (!STI.isVulkanEnv()) { 3369 unsigned Alignment = I.getOperand(2).getImm(); 3370 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3371 {Alignment}); 3372 } 3373 return Res; 3374 } 3375 3376 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3377 // InstructionSelector walks backwards through the instructions. We can use 3378 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3379 // first, so can generate an OpBranchConditional here. If there is no 3380 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3381 const MachineInstr *PrevI = I.getPrevNode(); 3382 MachineBasicBlock &MBB = *I.getParent(); 3383 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3384 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3385 .addUse(PrevI->getOperand(0).getReg()) 3386 .addMBB(PrevI->getOperand(1).getMBB()) 3387 .addMBB(I.getOperand(0).getMBB()) 3388 .constrainAllUses(TII, TRI, RBI); 3389 } 3390 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3391 .addMBB(I.getOperand(0).getMBB()) 3392 .constrainAllUses(TII, TRI, RBI); 3393 } 3394 3395 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3396 // InstructionSelector walks backwards through the instructions. For an 3397 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3398 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3399 // generate the OpBranchConditional in selectBranch above. 3400 // 3401 // If an OpBranchConditional has been generated, we simply return, as the work 3402 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3403 // implicit fallthrough to the next basic block, so we need to create an 3404 // OpBranchConditional with an explicit "false" argument pointing to the next 3405 // basic block that LLVM would fall through to. 3406 const MachineInstr *NextI = I.getNextNode(); 3407 // Check if this has already been successfully selected. 3408 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3409 return true; 3410 // Must be relying on implicit block fallthrough, so generate an 3411 // OpBranchConditional with the "next" basic block as the "false" target. 3412 MachineBasicBlock &MBB = *I.getParent(); 3413 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3414 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3415 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3416 .addUse(I.getOperand(0).getReg()) 3417 .addMBB(I.getOperand(1).getMBB()) 3418 .addMBB(NextMBB) 3419 .constrainAllUses(TII, TRI, RBI); 3420 } 3421 3422 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3423 const SPIRVType *ResType, 3424 MachineInstr &I) const { 3425 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3426 .addDef(ResVReg) 3427 .addUse(GR.getSPIRVTypeID(ResType)); 3428 const unsigned NumOps = I.getNumOperands(); 3429 for (unsigned i = 1; i < NumOps; i += 2) { 3430 MIB.addUse(I.getOperand(i + 0).getReg()); 3431 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3432 } 3433 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3434 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3435 MIB->removeOperand(1); 3436 return Res; 3437 } 3438 3439 bool SPIRVInstructionSelector::selectGlobalValue( 3440 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3441 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3442 MachineIRBuilder MIRBuilder(I); 3443 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3444 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3445 SPIRVType *PointerBaseType; 3446 if (GVType->isArrayTy()) { 3447 SPIRVType *ArrayElementType = 3448 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3449 SPIRV::AccessQualifier::ReadWrite, false); 3450 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3451 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3452 } else { 3453 PointerBaseType = GR.getOrCreateSPIRVType( 3454 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3455 } 3456 3457 std::string GlobalIdent; 3458 if (!GV->hasName()) { 3459 unsigned &ID = UnnamedGlobalIDs[GV]; 3460 if (ID == 0) 3461 ID = UnnamedGlobalIDs.size(); 3462 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3463 } else { 3464 GlobalIdent = GV->getName(); 3465 } 3466 3467 // Behaviour of functions as operands depends on availability of the 3468 // corresponding extension (SPV_INTEL_function_pointers): 3469 // - If there is an extension to operate with functions as operands: 3470 // We create a proper constant operand and evaluate a correct type for a 3471 // function pointer. 3472 // - Without the required extension: 3473 // We have functions as operands in tests with blocks of instruction e.g. in 3474 // transcoding/global_block.ll. These operands are not used and should be 3475 // substituted by zero constants. Their type is expected to be always 3476 // OpTypePointer Function %uchar. 3477 if (isa<Function>(GV)) { 3478 const Constant *ConstVal = GV; 3479 MachineBasicBlock &BB = *I.getParent(); 3480 Register NewReg = GR.find(ConstVal, GR.CurMF); 3481 if (!NewReg.isValid()) { 3482 Register NewReg = ResVReg; 3483 GR.add(ConstVal, GR.CurMF, NewReg); 3484 const Function *GVFun = 3485 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3486 ? dyn_cast<Function>(GV) 3487 : nullptr; 3488 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3489 PointerBaseType, I, TII, 3490 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3491 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3492 if (GVFun) { 3493 // References to a function via function pointers generate virtual 3494 // registers without a definition. We will resolve it later, during 3495 // module analysis stage. 3496 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 3497 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3498 Register FuncVReg = 3499 MRI->createGenericVirtualRegister(GR.getRegType(ResType)); 3500 MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); 3501 MachineInstrBuilder MIB1 = 3502 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 3503 .addDef(FuncVReg) 3504 .addUse(ResTypeReg); 3505 MachineInstrBuilder MIB2 = 3506 BuildMI(BB, I, I.getDebugLoc(), 3507 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3508 .addDef(NewReg) 3509 .addUse(ResTypeReg) 3510 .addUse(FuncVReg); 3511 // mapping the function pointer to the used Function 3512 GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); 3513 return MIB1.constrainAllUses(TII, TRI, RBI) && 3514 MIB2.constrainAllUses(TII, TRI, RBI); 3515 } 3516 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3517 .addDef(NewReg) 3518 .addUse(GR.getSPIRVTypeID(ResType)) 3519 .constrainAllUses(TII, TRI, RBI); 3520 } 3521 assert(NewReg != ResVReg); 3522 return BuildCOPY(ResVReg, NewReg, I); 3523 } 3524 auto GlobalVar = cast<GlobalVariable>(GV); 3525 assert(GlobalVar->getName() != "llvm.global.annotations"); 3526 3527 // Skip empty declaration for GVs with initializers till we get the decl with 3528 // passed initializer. 3529 if (hasInitializer(GlobalVar) && !Init) 3530 return true; 3531 3532 bool HasLnkTy = !GV->hasInternalLinkage() && !GV->hasPrivateLinkage(); 3533 SPIRV::LinkageType::LinkageType LnkType = 3534 GV->isDeclarationForLinker() 3535 ? SPIRV::LinkageType::Import 3536 : (GV->hasLinkOnceODRLinkage() && 3537 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3538 ? SPIRV::LinkageType::LinkOnceODR 3539 : SPIRV::LinkageType::Export); 3540 3541 const unsigned AddrSpace = GV->getAddressSpace(); 3542 SPIRV::StorageClass::StorageClass StorageClass = 3543 addressSpaceToStorageClass(AddrSpace, STI); 3544 SPIRVType *ResType = 3545 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3546 Register Reg = GR.buildGlobalVariable( 3547 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3548 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3549 return Reg.isValid(); 3550 } 3551 3552 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3553 const SPIRVType *ResType, 3554 MachineInstr &I) const { 3555 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3556 return selectExtInst(ResVReg, ResType, I, CL::log10); 3557 } 3558 3559 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3560 // is implemented as: 3561 // log10(x) = log2(x) * (1 / log2(10)) 3562 // = log2(x) * 0.30103 3563 3564 MachineIRBuilder MIRBuilder(I); 3565 MachineBasicBlock &BB = *I.getParent(); 3566 3567 // Build log2(x). 3568 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3569 bool Result = 3570 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3571 .addDef(VarReg) 3572 .addUse(GR.getSPIRVTypeID(ResType)) 3573 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3574 .addImm(GL::Log2) 3575 .add(I.getOperand(1)) 3576 .constrainAllUses(TII, TRI, RBI); 3577 3578 // Build 0.30103. 3579 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3580 ResType->getOpcode() == SPIRV::OpTypeFloat); 3581 // TODO: Add matrix implementation once supported by the HLSL frontend. 3582 const SPIRVType *SpirvScalarType = 3583 ResType->getOpcode() == SPIRV::OpTypeVector 3584 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3585 : ResType; 3586 Register ScaleReg = 3587 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3588 3589 // Multiply log2(x) by 0.30103 to get log10(x) result. 3590 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3591 ? SPIRV::OpVectorTimesScalar 3592 : SPIRV::OpFMulS; 3593 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3594 .addDef(ResVReg) 3595 .addUse(GR.getSPIRVTypeID(ResType)) 3596 .addUse(VarReg) 3597 .addUse(ScaleReg) 3598 .constrainAllUses(TII, TRI, RBI); 3599 } 3600 3601 // Generate the instructions to load 3-element vector builtin input 3602 // IDs/Indices. 3603 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3604 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3605 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3606 const SPIRVType *ResType, MachineInstr &I) const { 3607 MachineIRBuilder MIRBuilder(I); 3608 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3609 const SPIRVType *Vec3Ty = 3610 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3611 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3612 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3613 3614 // Create new register for the input ID builtin variable. 3615 Register NewRegister = 3616 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3617 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3618 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3619 3620 // Build global variable with the necessary decorations for the input ID 3621 // builtin variable. 3622 Register Variable = GR.buildGlobalVariable( 3623 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3624 SPIRV::StorageClass::Input, nullptr, true, true, 3625 SPIRV::LinkageType::Import, MIRBuilder, false); 3626 3627 // Create new register for loading value. 3628 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3629 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3630 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3631 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3632 3633 // Load v3uint value from the global variable. 3634 bool Result = 3635 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3636 .addDef(LoadedRegister) 3637 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3638 .addUse(Variable); 3639 3640 // Get the input ID index. Expecting operand is a constant immediate value, 3641 // wrapped in a type assignment. 3642 assert(I.getOperand(2).isReg()); 3643 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3644 3645 // Extract the input ID from the loaded vector value. 3646 MachineBasicBlock &BB = *I.getParent(); 3647 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3648 .addDef(ResVReg) 3649 .addUse(GR.getSPIRVTypeID(ResType)) 3650 .addUse(LoadedRegister) 3651 .addImm(ThreadId); 3652 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3653 } 3654 3655 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3656 MachineInstr &I) const { 3657 MachineIRBuilder MIRBuilder(I); 3658 if (Type->getOpcode() != SPIRV::OpTypeVector) 3659 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3660 3661 uint64_t VectorSize = Type->getOperand(2).getImm(); 3662 if (VectorSize == 4) 3663 return Type; 3664 3665 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3666 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3667 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3668 } 3669 3670 namespace llvm { 3671 InstructionSelector * 3672 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3673 const SPIRVSubtarget &Subtarget, 3674 const RegisterBankInfo &RBI) { 3675 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3676 } 3677 } // namespace llvm 3678