1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 37 #define DEBUG_TYPE "spirv-isel" 38 39 using namespace llvm; 40 namespace CL = SPIRV::OpenCLExtInst; 41 namespace GL = SPIRV::GLSLExtInst; 42 43 using ExtInstList = 44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 45 46 namespace { 47 48 #define GET_GLOBALISEL_PREDICATE_BITSET 49 #include "SPIRVGenGlobalISel.inc" 50 #undef GET_GLOBALISEL_PREDICATE_BITSET 51 52 class SPIRVInstructionSelector : public InstructionSelector { 53 const SPIRVSubtarget &STI; 54 const SPIRVInstrInfo &TII; 55 const SPIRVRegisterInfo &TRI; 56 const RegisterBankInfo &RBI; 57 SPIRVGlobalRegistry &GR; 58 MachineRegisterInfo *MRI; 59 MachineFunction *HasVRegsReset = nullptr; 60 61 /// We need to keep track of the number we give to anonymous global values to 62 /// generate the same name every time when this is needed. 63 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 64 SmallPtrSet<MachineInstr *, 8> DeadMIs; 65 66 public: 67 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 68 const SPIRVSubtarget &ST, 69 const RegisterBankInfo &RBI); 70 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 71 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 72 BlockFrequencyInfo *BFI) override; 73 // Common selection code. Instruction-specific selection occurs in spvSelect. 74 bool select(MachineInstr &I) override; 75 static const char *getName() { return DEBUG_TYPE; } 76 77 #define GET_GLOBALISEL_PREDICATES_DECL 78 #include "SPIRVGenGlobalISel.inc" 79 #undef GET_GLOBALISEL_PREDICATES_DECL 80 81 #define GET_GLOBALISEL_TEMPORARIES_DECL 82 #include "SPIRVGenGlobalISel.inc" 83 #undef GET_GLOBALISEL_TEMPORARIES_DECL 84 85 private: 86 void resetVRegsType(MachineFunction &MF); 87 88 // tblgen-erated 'select' implementation, used as the initial selector for 89 // the patterns that don't require complex C++. 90 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 91 92 // All instruction-specific selection that didn't happen in "select()". 93 // Is basically a large Switch/Case delegating to all other select method. 94 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 95 MachineInstr &I) const; 96 97 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 98 MachineInstr &I, bool IsSigned) const; 99 100 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 101 MachineInstr &I, bool IsSigned) const; 102 103 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 104 MachineInstr &I, Register SrcReg, 105 bool IsSigned) const; 106 107 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 108 MachineInstr &I, bool IsSigned) const; 109 110 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 111 const MachineInstr *Init = nullptr) const; 112 113 bool selectOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 114 MachineInstr &I, std::vector<Register> SrcRegs, 115 unsigned Opcode) const; 116 117 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 118 unsigned Opcode) const; 119 120 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 121 MachineInstr &I) const; 122 123 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I) const; 125 bool selectStore(MachineInstr &I) const; 126 127 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I) const; 129 bool selectStackRestore(MachineInstr &I) const; 130 131 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 132 133 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 134 MachineInstr &I, unsigned NewOpcode, 135 unsigned NegateOpcode = 0) const; 136 137 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 138 MachineInstr &I) const; 139 140 bool selectFence(MachineInstr &I) const; 141 142 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 143 MachineInstr &I) const; 144 145 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I, unsigned OpType) const; 147 148 bool selectAll(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 151 bool selectAny(Register ResVReg, const SPIRVType *ResType, 152 MachineInstr &I) const; 153 154 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 155 MachineInstr &I) const; 156 157 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 158 MachineInstr &I) const; 159 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 162 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 163 unsigned comparisonOpcode, MachineInstr &I) const; 164 bool selectCross(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 bool selectDiscard(Register ResVReg, const SPIRVType *ResType, 167 MachineInstr &I) const; 168 169 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I) const; 171 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 172 MachineInstr &I) const; 173 174 bool selectSign(Register ResVReg, const SPIRVType *ResType, 175 MachineInstr &I) const; 176 177 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 178 MachineInstr &I) const; 179 180 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 181 MachineInstr &I, unsigned Opcode) const; 182 183 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 184 MachineInstr &I, bool Signed) const; 185 186 bool selectIntegerDotExpansion(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 189 template <bool Signed> 190 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 191 MachineInstr &I) const; 192 template <bool Signed> 193 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 194 MachineInstr &I) const; 195 196 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 197 int OpIdx) const; 198 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 199 int OpIdx) const; 200 201 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 202 MachineInstr &I) const; 203 204 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 205 bool IsSigned) const; 206 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 207 bool IsSigned, unsigned Opcode) const; 208 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 209 bool IsSigned) const; 210 211 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 212 MachineInstr &I) const; 213 214 bool selectSUCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 215 bool IsSigned) const; 216 217 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 218 const SPIRVType *intTy, const SPIRVType *boolTy) const; 219 220 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I) const; 224 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 227 MachineInstr &I) const; 228 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 229 MachineInstr &I) const; 230 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 233 MachineInstr &I) const; 234 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 235 MachineInstr &I) const; 236 237 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 240 MachineInstr &I) const; 241 242 bool selectBranch(MachineInstr &I) const; 243 bool selectBranchCond(MachineInstr &I) const; 244 245 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 246 MachineInstr &I) const; 247 248 [[maybe_unused]] bool selectExtInst(Register ResVReg, 249 const SPIRVType *RestType, 250 MachineInstr &I, 251 GL::GLSLExtInst GLInst) const; 252 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 253 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 254 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I, CL::OpenCLExtInst CLInst, 256 GL::GLSLExtInst GLInst) const; 257 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I, const ExtInstList &ExtInsts) const; 259 260 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 261 MachineInstr &I) const; 262 263 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 264 MachineInstr &I) const; 265 266 bool selectWaveOpInst(Register ResVReg, const SPIRVType *ResType, 267 MachineInstr &I, unsigned Opcode) const; 268 269 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 270 MachineInstr &I) const; 271 272 bool selectUnmergeValues(MachineInstr &I) const; 273 274 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 275 MachineInstr &I) const; 276 277 void selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 278 MachineInstr &I) const; 279 280 void selectImageWriteIntrinsic(MachineInstr &I) const; 281 282 // Utilities 283 std::pair<Register, bool> 284 buildI32Constant(uint32_t Val, MachineInstr &I, 285 const SPIRVType *ResType = nullptr) const; 286 287 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 288 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 289 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 290 MachineInstr &I) const; 291 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 292 293 bool wrapIntoSpecConstantOp(MachineInstr &I, 294 SmallVector<Register> &CompositeArgs) const; 295 296 Register getUcharPtrTypeReg(MachineInstr &I, 297 SPIRV::StorageClass::StorageClass SC) const; 298 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 299 Register Src, Register DestType, 300 uint32_t Opcode) const; 301 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 302 SPIRVType *SrcPtrTy) const; 303 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 304 uint32_t Binding, uint32_t ArraySize, 305 Register IndexReg, bool IsNonUniform, 306 MachineIRBuilder MIRBuilder) const; 307 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 308 void extractSubvector(Register &ResVReg, const SPIRVType *ResType, 309 Register &ReadReg, MachineInstr &InsertionPoint) const; 310 bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; 311 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, 312 Register ResVReg, const SPIRVType *ResType, 313 MachineInstr &I) const; 314 }; 315 316 } // end anonymous namespace 317 318 #define GET_GLOBALISEL_IMPL 319 #include "SPIRVGenGlobalISel.inc" 320 #undef GET_GLOBALISEL_IMPL 321 322 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 323 const SPIRVSubtarget &ST, 324 const RegisterBankInfo &RBI) 325 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 326 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 327 #define GET_GLOBALISEL_PREDICATES_INIT 328 #include "SPIRVGenGlobalISel.inc" 329 #undef GET_GLOBALISEL_PREDICATES_INIT 330 #define GET_GLOBALISEL_TEMPORARIES_INIT 331 #include "SPIRVGenGlobalISel.inc" 332 #undef GET_GLOBALISEL_TEMPORARIES_INIT 333 { 334 } 335 336 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 337 CodeGenCoverage *CoverageInfo, 338 ProfileSummaryInfo *PSI, 339 BlockFrequencyInfo *BFI) { 340 MRI = &MF.getRegInfo(); 341 GR.setCurrentFunc(MF); 342 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 343 } 344 345 // Ensure that register classes correspond to pattern matching rules. 346 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 347 if (HasVRegsReset == &MF) 348 return; 349 HasVRegsReset = &MF; 350 351 MachineRegisterInfo &MRI = MF.getRegInfo(); 352 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 353 Register Reg = Register::index2VirtReg(I); 354 LLT RegType = MRI.getType(Reg); 355 if (RegType.isScalar()) 356 MRI.setType(Reg, LLT::scalar(64)); 357 else if (RegType.isPointer()) 358 MRI.setType(Reg, LLT::pointer(0, 64)); 359 else if (RegType.isVector()) 360 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 361 } 362 for (const auto &MBB : MF) { 363 for (const auto &MI : MBB) { 364 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 365 continue; 366 Register DstReg = MI.getOperand(0).getReg(); 367 LLT DstType = MRI.getType(DstReg); 368 Register SrcReg = MI.getOperand(1).getReg(); 369 LLT SrcType = MRI.getType(SrcReg); 370 if (DstType != SrcType) 371 MRI.setType(DstReg, MRI.getType(SrcReg)); 372 373 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 374 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 375 if (DstRC != SrcRC && SrcRC) 376 MRI.setRegClass(DstReg, SrcRC); 377 } 378 } 379 } 380 381 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 382 383 // Defined in SPIRVLegalizerInfo.cpp. 384 extern bool isTypeFoldingSupported(unsigned Opcode); 385 386 bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { 387 for (const auto &MO : MI.all_defs()) { 388 Register Reg = MO.getReg(); 389 if (Reg.isPhysical() || !MRI.use_nodbg_empty(Reg)) 390 return false; 391 } 392 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || 393 MI.isLifetimeMarker()) 394 return false; 395 if (MI.isPHI()) 396 return true; 397 if (MI.mayStore() || MI.isCall() || 398 (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || 399 MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) 400 return false; 401 return true; 402 } 403 404 bool SPIRVInstructionSelector::select(MachineInstr &I) { 405 resetVRegsType(*I.getParent()->getParent()); 406 407 assert(I.getParent() && "Instruction should be in a basic block!"); 408 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 409 410 Register Opcode = I.getOpcode(); 411 // If it's not a GMIR instruction, we've selected it already. 412 if (!isPreISelGenericOpcode(Opcode)) { 413 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 414 Register DstReg = I.getOperand(0).getReg(); 415 Register SrcReg = I.getOperand(1).getReg(); 416 auto *Def = MRI->getVRegDef(SrcReg); 417 if (isTypeFoldingSupported(Def->getOpcode())) { 418 bool Res = selectImpl(I, *CoverageInfo); 419 LLVM_DEBUG({ 420 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 421 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 422 I.print(dbgs()); 423 } 424 }); 425 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 426 if (Res) { 427 if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) 428 DeadMIs.insert(Def); 429 return Res; 430 } 431 } 432 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 433 MRI->replaceRegWith(SrcReg, DstReg); 434 I.removeFromParent(); 435 return true; 436 } else if (I.getNumDefs() == 1) { 437 // Make all vregs 64 bits (for SPIR-V IDs). 438 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 439 } 440 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 441 } 442 443 if (DeadMIs.contains(&I)) { 444 // if the instruction has been already made dead by folding it away 445 // erase it 446 LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); 447 salvageDebugInfo(*MRI, I); 448 I.eraseFromParent(); 449 return true; 450 } 451 452 if (I.getNumOperands() != I.getNumExplicitOperands()) { 453 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 454 return false; 455 } 456 457 // Common code for getting return reg+type, and removing selected instr 458 // from parent occurs here. Instr-specific selection happens in spvSelect(). 459 bool HasDefs = I.getNumDefs() > 0; 460 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 461 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 462 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 463 if (spvSelect(ResVReg, ResType, I)) { 464 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 465 for (unsigned i = 0; i < I.getNumDefs(); ++i) 466 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 467 I.removeFromParent(); 468 return true; 469 } 470 return false; 471 } 472 473 static bool mayApplyGenericSelection(unsigned Opcode) { 474 switch (Opcode) { 475 case TargetOpcode::G_CONSTANT: 476 return false; 477 case TargetOpcode::G_SADDO: 478 case TargetOpcode::G_SSUBO: 479 return true; 480 } 481 return isTypeFoldingSupported(Opcode); 482 } 483 484 bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, 485 MachineInstr &I) const { 486 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); 487 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 488 if (DstRC != SrcRC && SrcRC) 489 MRI->setRegClass(DestReg, SrcRC); 490 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 491 TII.get(TargetOpcode::COPY)) 492 .addDef(DestReg) 493 .addUse(SrcReg) 494 .constrainAllUses(TII, TRI, RBI); 495 } 496 497 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 498 const SPIRVType *ResType, 499 MachineInstr &I) const { 500 const unsigned Opcode = I.getOpcode(); 501 if (mayApplyGenericSelection(Opcode)) 502 return selectImpl(I, *CoverageInfo); 503 switch (Opcode) { 504 case TargetOpcode::G_CONSTANT: 505 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 506 I); 507 case TargetOpcode::G_GLOBAL_VALUE: 508 return selectGlobalValue(ResVReg, I); 509 case TargetOpcode::G_IMPLICIT_DEF: 510 return selectOpUndef(ResVReg, ResType, I); 511 case TargetOpcode::G_FREEZE: 512 return selectFreeze(ResVReg, ResType, I); 513 514 case TargetOpcode::G_INTRINSIC: 515 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 516 case TargetOpcode::G_INTRINSIC_CONVERGENT: 517 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 518 return selectIntrinsic(ResVReg, ResType, I); 519 case TargetOpcode::G_BITREVERSE: 520 return selectBitreverse(ResVReg, ResType, I); 521 522 case TargetOpcode::G_BUILD_VECTOR: 523 return selectBuildVector(ResVReg, ResType, I); 524 case TargetOpcode::G_SPLAT_VECTOR: 525 return selectSplatVector(ResVReg, ResType, I); 526 527 case TargetOpcode::G_SHUFFLE_VECTOR: { 528 MachineBasicBlock &BB = *I.getParent(); 529 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 530 .addDef(ResVReg) 531 .addUse(GR.getSPIRVTypeID(ResType)) 532 .addUse(I.getOperand(1).getReg()) 533 .addUse(I.getOperand(2).getReg()); 534 for (auto V : I.getOperand(3).getShuffleMask()) 535 MIB.addImm(V); 536 return MIB.constrainAllUses(TII, TRI, RBI); 537 } 538 case TargetOpcode::G_MEMMOVE: 539 case TargetOpcode::G_MEMCPY: 540 case TargetOpcode::G_MEMSET: 541 return selectMemOperation(ResVReg, I); 542 543 case TargetOpcode::G_ICMP: 544 return selectICmp(ResVReg, ResType, I); 545 case TargetOpcode::G_FCMP: 546 return selectFCmp(ResVReg, ResType, I); 547 548 case TargetOpcode::G_FRAME_INDEX: 549 return selectFrameIndex(ResVReg, ResType, I); 550 551 case TargetOpcode::G_LOAD: 552 return selectLoad(ResVReg, ResType, I); 553 case TargetOpcode::G_STORE: 554 return selectStore(I); 555 556 case TargetOpcode::G_BR: 557 return selectBranch(I); 558 case TargetOpcode::G_BRCOND: 559 return selectBranchCond(I); 560 561 case TargetOpcode::G_PHI: 562 return selectPhi(ResVReg, ResType, I); 563 564 case TargetOpcode::G_FPTOSI: 565 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 566 case TargetOpcode::G_FPTOUI: 567 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 568 569 case TargetOpcode::G_SITOFP: 570 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 571 case TargetOpcode::G_UITOFP: 572 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 573 574 case TargetOpcode::G_CTPOP: 575 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 576 case TargetOpcode::G_SMIN: 577 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 578 case TargetOpcode::G_UMIN: 579 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 580 581 case TargetOpcode::G_SMAX: 582 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 583 case TargetOpcode::G_UMAX: 584 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 585 586 case TargetOpcode::G_SCMP: 587 return selectSUCmp(ResVReg, ResType, I, true); 588 case TargetOpcode::G_UCMP: 589 return selectSUCmp(ResVReg, ResType, I, false); 590 591 case TargetOpcode::G_STRICT_FMA: 592 case TargetOpcode::G_FMA: 593 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 594 595 case TargetOpcode::G_STRICT_FLDEXP: 596 return selectExtInst(ResVReg, ResType, I, CL::ldexp); 597 598 case TargetOpcode::G_FPOW: 599 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 600 case TargetOpcode::G_FPOWI: 601 return selectExtInst(ResVReg, ResType, I, CL::pown); 602 603 case TargetOpcode::G_FEXP: 604 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 605 case TargetOpcode::G_FEXP2: 606 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 607 608 case TargetOpcode::G_FLOG: 609 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 610 case TargetOpcode::G_FLOG2: 611 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 612 case TargetOpcode::G_FLOG10: 613 return selectLog10(ResVReg, ResType, I); 614 615 case TargetOpcode::G_FABS: 616 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 617 case TargetOpcode::G_ABS: 618 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 619 620 case TargetOpcode::G_FMINNUM: 621 case TargetOpcode::G_FMINIMUM: 622 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 623 case TargetOpcode::G_FMAXNUM: 624 case TargetOpcode::G_FMAXIMUM: 625 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 626 627 case TargetOpcode::G_FCOPYSIGN: 628 return selectExtInst(ResVReg, ResType, I, CL::copysign); 629 630 case TargetOpcode::G_FCEIL: 631 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 632 case TargetOpcode::G_FFLOOR: 633 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 634 635 case TargetOpcode::G_FCOS: 636 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 637 case TargetOpcode::G_FSIN: 638 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 639 case TargetOpcode::G_FTAN: 640 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 641 case TargetOpcode::G_FACOS: 642 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 643 case TargetOpcode::G_FASIN: 644 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 645 case TargetOpcode::G_FATAN: 646 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 647 case TargetOpcode::G_FATAN2: 648 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 649 case TargetOpcode::G_FCOSH: 650 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 651 case TargetOpcode::G_FSINH: 652 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 653 case TargetOpcode::G_FTANH: 654 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 655 656 case TargetOpcode::G_STRICT_FSQRT: 657 case TargetOpcode::G_FSQRT: 658 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 659 660 case TargetOpcode::G_CTTZ: 661 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 662 return selectExtInst(ResVReg, ResType, I, CL::ctz); 663 case TargetOpcode::G_CTLZ: 664 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 665 return selectExtInst(ResVReg, ResType, I, CL::clz); 666 667 case TargetOpcode::G_INTRINSIC_ROUND: 668 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 669 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 670 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 671 case TargetOpcode::G_INTRINSIC_TRUNC: 672 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 673 case TargetOpcode::G_FRINT: 674 case TargetOpcode::G_FNEARBYINT: 675 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 676 677 case TargetOpcode::G_SMULH: 678 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 679 case TargetOpcode::G_UMULH: 680 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 681 682 case TargetOpcode::G_SADDSAT: 683 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 684 case TargetOpcode::G_UADDSAT: 685 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 686 case TargetOpcode::G_SSUBSAT: 687 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 688 case TargetOpcode::G_USUBSAT: 689 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 690 691 case TargetOpcode::G_UADDO: 692 return selectOverflowArith(ResVReg, ResType, I, 693 ResType->getOpcode() == SPIRV::OpTypeVector 694 ? SPIRV::OpIAddCarryV 695 : SPIRV::OpIAddCarryS); 696 case TargetOpcode::G_USUBO: 697 return selectOverflowArith(ResVReg, ResType, I, 698 ResType->getOpcode() == SPIRV::OpTypeVector 699 ? SPIRV::OpISubBorrowV 700 : SPIRV::OpISubBorrowS); 701 case TargetOpcode::G_UMULO: 702 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 703 case TargetOpcode::G_SMULO: 704 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 705 706 case TargetOpcode::G_SEXT: 707 return selectExt(ResVReg, ResType, I, true); 708 case TargetOpcode::G_ANYEXT: 709 case TargetOpcode::G_ZEXT: 710 return selectExt(ResVReg, ResType, I, false); 711 case TargetOpcode::G_TRUNC: 712 return selectTrunc(ResVReg, ResType, I); 713 case TargetOpcode::G_FPTRUNC: 714 case TargetOpcode::G_FPEXT: 715 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 716 717 case TargetOpcode::G_PTRTOINT: 718 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 719 case TargetOpcode::G_INTTOPTR: 720 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 721 case TargetOpcode::G_BITCAST: 722 return selectBitcast(ResVReg, ResType, I); 723 case TargetOpcode::G_ADDRSPACE_CAST: 724 return selectAddrSpaceCast(ResVReg, ResType, I); 725 case TargetOpcode::G_PTR_ADD: { 726 // Currently, we get G_PTR_ADD only applied to global variables. 727 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 728 Register GV = I.getOperand(1).getReg(); 729 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 730 (void)II; 731 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 732 (*II).getOpcode() == TargetOpcode::COPY || 733 (*II).getOpcode() == SPIRV::OpVariable) && 734 isImm(I.getOperand(2), MRI)); 735 // It may be the initialization of a global variable. 736 bool IsGVInit = false; 737 for (MachineRegisterInfo::use_instr_iterator 738 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 739 UseEnd = MRI->use_instr_end(); 740 UseIt != UseEnd; UseIt = std::next(UseIt)) { 741 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 742 (*UseIt).getOpcode() == SPIRV::OpVariable) { 743 IsGVInit = true; 744 break; 745 } 746 } 747 MachineBasicBlock &BB = *I.getParent(); 748 if (!IsGVInit) { 749 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 750 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 751 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 752 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 753 // Build a new virtual register that is associated with the required 754 // data type. 755 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 756 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 757 // Having a correctly typed base we are ready to build the actually 758 // required GEP. It may not be a constant though, because all Operands 759 // of OpSpecConstantOp is to originate from other const instructions, 760 // and only the AccessChain named opcodes accept a global OpVariable 761 // instruction. We can't use an AccessChain opcode because of the type 762 // mismatch between result and base types. 763 if (!GR.isBitcastCompatible(ResType, GVType)) 764 report_fatal_error( 765 "incompatible result and operand types in a bitcast"); 766 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 767 MachineInstrBuilder MIB = 768 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 769 .addDef(NewVReg) 770 .addUse(ResTypeReg) 771 .addUse(GV); 772 return MIB.constrainAllUses(TII, TRI, RBI) && 773 BuildMI(BB, I, I.getDebugLoc(), 774 TII.get(STI.isVulkanEnv() 775 ? SPIRV::OpInBoundsAccessChain 776 : SPIRV::OpInBoundsPtrAccessChain)) 777 .addDef(ResVReg) 778 .addUse(ResTypeReg) 779 .addUse(NewVReg) 780 .addUse(I.getOperand(2).getReg()) 781 .constrainAllUses(TII, TRI, RBI); 782 } else { 783 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 784 .addDef(ResVReg) 785 .addUse(GR.getSPIRVTypeID(ResType)) 786 .addImm( 787 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 788 .addUse(GV) 789 .addUse(I.getOperand(2).getReg()) 790 .constrainAllUses(TII, TRI, RBI); 791 } 792 } 793 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 794 // initialize a global variable with a constant expression (e.g., the test 795 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 796 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 797 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 798 .addDef(ResVReg) 799 .addUse(GR.getSPIRVTypeID(ResType)) 800 .addImm(static_cast<uint32_t>( 801 SPIRV::Opcode::InBoundsPtrAccessChain)) 802 .addUse(GV) 803 .addUse(Idx) 804 .addUse(I.getOperand(2).getReg()); 805 return MIB.constrainAllUses(TII, TRI, RBI); 806 } 807 808 case TargetOpcode::G_ATOMICRMW_OR: 809 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 810 case TargetOpcode::G_ATOMICRMW_ADD: 811 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 812 case TargetOpcode::G_ATOMICRMW_AND: 813 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 814 case TargetOpcode::G_ATOMICRMW_MAX: 815 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 816 case TargetOpcode::G_ATOMICRMW_MIN: 817 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 818 case TargetOpcode::G_ATOMICRMW_SUB: 819 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 820 case TargetOpcode::G_ATOMICRMW_XOR: 821 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 822 case TargetOpcode::G_ATOMICRMW_UMAX: 823 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 824 case TargetOpcode::G_ATOMICRMW_UMIN: 825 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 826 case TargetOpcode::G_ATOMICRMW_XCHG: 827 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 828 case TargetOpcode::G_ATOMIC_CMPXCHG: 829 return selectAtomicCmpXchg(ResVReg, ResType, I); 830 831 case TargetOpcode::G_ATOMICRMW_FADD: 832 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 833 case TargetOpcode::G_ATOMICRMW_FSUB: 834 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 835 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 836 SPIRV::OpFNegate); 837 case TargetOpcode::G_ATOMICRMW_FMIN: 838 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 839 case TargetOpcode::G_ATOMICRMW_FMAX: 840 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 841 842 case TargetOpcode::G_FENCE: 843 return selectFence(I); 844 845 case TargetOpcode::G_STACKSAVE: 846 return selectStackSave(ResVReg, ResType, I); 847 case TargetOpcode::G_STACKRESTORE: 848 return selectStackRestore(I); 849 850 case TargetOpcode::G_UNMERGE_VALUES: 851 return selectUnmergeValues(I); 852 853 // Discard gen opcodes for intrinsics which we do not expect to actually 854 // represent code after lowering or intrinsics which are not implemented but 855 // should not crash when found in a customer's LLVM IR input. 856 case TargetOpcode::G_TRAP: 857 case TargetOpcode::G_DEBUGTRAP: 858 case TargetOpcode::G_UBSANTRAP: 859 case TargetOpcode::DBG_LABEL: 860 return true; 861 862 default: 863 return false; 864 } 865 } 866 867 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 868 const SPIRVType *ResType, 869 MachineInstr &I, 870 GL::GLSLExtInst GLInst) const { 871 return selectExtInst(ResVReg, ResType, I, 872 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 873 } 874 875 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 876 const SPIRVType *ResType, 877 MachineInstr &I, 878 CL::OpenCLExtInst CLInst) const { 879 return selectExtInst(ResVReg, ResType, I, 880 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 881 } 882 883 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 884 const SPIRVType *ResType, 885 MachineInstr &I, 886 CL::OpenCLExtInst CLInst, 887 GL::GLSLExtInst GLInst) const { 888 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 889 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 890 return selectExtInst(ResVReg, ResType, I, ExtInsts); 891 } 892 893 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 894 const SPIRVType *ResType, 895 MachineInstr &I, 896 const ExtInstList &Insts) const { 897 898 for (const auto &Ex : Insts) { 899 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 900 uint32_t Opcode = Ex.second; 901 if (STI.canUseExtInstSet(Set)) { 902 MachineBasicBlock &BB = *I.getParent(); 903 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 904 .addDef(ResVReg) 905 .addUse(GR.getSPIRVTypeID(ResType)) 906 .addImm(static_cast<uint32_t>(Set)) 907 .addImm(Opcode); 908 const unsigned NumOps = I.getNumOperands(); 909 unsigned Index = 1; 910 if (Index < NumOps && 911 I.getOperand(Index).getType() == 912 MachineOperand::MachineOperandType::MO_IntrinsicID) 913 Index = 2; 914 for (; Index < NumOps; ++Index) 915 MIB.add(I.getOperand(Index)); 916 return MIB.constrainAllUses(TII, TRI, RBI); 917 } 918 } 919 return false; 920 } 921 922 bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, 923 const SPIRVType *ResType, 924 MachineInstr &I, 925 std::vector<Register> Srcs, 926 unsigned Opcode) const { 927 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 928 .addDef(ResVReg) 929 .addUse(GR.getSPIRVTypeID(ResType)); 930 for (Register SReg : Srcs) { 931 MIB.addUse(SReg); 932 } 933 return MIB.constrainAllUses(TII, TRI, RBI); 934 } 935 936 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 937 const SPIRVType *ResType, 938 MachineInstr &I, 939 unsigned Opcode) const { 940 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 941 Register SrcReg = I.getOperand(1).getReg(); 942 bool IsGV = false; 943 for (MachineRegisterInfo::def_instr_iterator DefIt = 944 MRI->def_instr_begin(SrcReg); 945 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 946 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 947 IsGV = true; 948 break; 949 } 950 } 951 if (IsGV) { 952 uint32_t SpecOpcode = 0; 953 switch (Opcode) { 954 case SPIRV::OpConvertPtrToU: 955 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 956 break; 957 case SPIRV::OpConvertUToPtr: 958 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 959 break; 960 } 961 if (SpecOpcode) 962 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 963 TII.get(SPIRV::OpSpecConstantOp)) 964 .addDef(ResVReg) 965 .addUse(GR.getSPIRVTypeID(ResType)) 966 .addImm(SpecOpcode) 967 .addUse(SrcReg) 968 .constrainAllUses(TII, TRI, RBI); 969 } 970 } 971 return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, 972 Opcode); 973 } 974 975 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 976 const SPIRVType *ResType, 977 MachineInstr &I) const { 978 Register OpReg = I.getOperand(1).getReg(); 979 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 980 if (!GR.isBitcastCompatible(ResType, OpType)) 981 report_fatal_error("incompatible result and operand types in a bitcast"); 982 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 983 } 984 985 static void addMemoryOperands(MachineMemOperand *MemOp, 986 MachineInstrBuilder &MIB) { 987 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 988 if (MemOp->isVolatile()) 989 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 990 if (MemOp->isNonTemporal()) 991 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 992 if (MemOp->getAlign().value()) 993 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 994 995 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 996 MIB.addImm(SpvMemOp); 997 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 998 MIB.addImm(MemOp->getAlign().value()); 999 } 1000 } 1001 1002 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 1003 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 1004 if (Flags & MachineMemOperand::Flags::MOVolatile) 1005 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 1006 if (Flags & MachineMemOperand::Flags::MONonTemporal) 1007 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 1008 1009 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 1010 MIB.addImm(SpvMemOp); 1011 } 1012 1013 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 1014 const SPIRVType *ResType, 1015 MachineInstr &I) const { 1016 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1017 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1018 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 1019 .addDef(ResVReg) 1020 .addUse(GR.getSPIRVTypeID(ResType)) 1021 .addUse(Ptr); 1022 if (!I.getNumMemOperands()) { 1023 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1024 I.getOpcode() == 1025 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1026 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1027 } else { 1028 addMemoryOperands(*I.memoperands_begin(), MIB); 1029 } 1030 return MIB.constrainAllUses(TII, TRI, RBI); 1031 } 1032 1033 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 1034 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 1035 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 1036 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 1037 MachineBasicBlock &BB = *I.getParent(); 1038 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 1039 .addUse(Ptr) 1040 .addUse(StoreVal); 1041 if (!I.getNumMemOperands()) { 1042 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 1043 I.getOpcode() == 1044 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 1045 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 1046 } else { 1047 addMemoryOperands(*I.memoperands_begin(), MIB); 1048 } 1049 return MIB.constrainAllUses(TII, TRI, RBI); 1050 } 1051 1052 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1053 const SPIRVType *ResType, 1054 MachineInstr &I) const { 1055 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1056 report_fatal_error( 1057 "llvm.stacksave intrinsic: this instruction requires the following " 1058 "SPIR-V extension: SPV_INTEL_variable_length_array", 1059 false); 1060 MachineBasicBlock &BB = *I.getParent(); 1061 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1062 .addDef(ResVReg) 1063 .addUse(GR.getSPIRVTypeID(ResType)) 1064 .constrainAllUses(TII, TRI, RBI); 1065 } 1066 1067 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1068 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1069 report_fatal_error( 1070 "llvm.stackrestore intrinsic: this instruction requires the following " 1071 "SPIR-V extension: SPV_INTEL_variable_length_array", 1072 false); 1073 if (!I.getOperand(0).isReg()) 1074 return false; 1075 MachineBasicBlock &BB = *I.getParent(); 1076 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1077 .addUse(I.getOperand(0).getReg()) 1078 .constrainAllUses(TII, TRI, RBI); 1079 } 1080 1081 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1082 MachineInstr &I) const { 1083 MachineBasicBlock &BB = *I.getParent(); 1084 Register SrcReg = I.getOperand(1).getReg(); 1085 bool Result = true; 1086 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1087 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1088 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1089 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1090 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1091 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1092 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1093 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1094 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1095 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1096 Function &CurFunction = GR.CurMF->getFunction(); 1097 Type *LLVMArrTy = 1098 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1099 // Module takes ownership of the global var. 1100 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1101 true, GlobalValue::InternalLinkage, 1102 Constant::getNullValue(LLVMArrTy)); 1103 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1104 GR.add(GV, GR.CurMF, VarReg); 1105 1106 Result &= 1107 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1108 .addDef(VarReg) 1109 .addUse(GR.getSPIRVTypeID(VarTy)) 1110 .addImm(SPIRV::StorageClass::UniformConstant) 1111 .addUse(Const) 1112 .constrainAllUses(TII, TRI, RBI); 1113 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1114 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1115 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1116 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1117 selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast); 1118 } 1119 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1120 .addUse(I.getOperand(0).getReg()) 1121 .addUse(SrcReg) 1122 .addUse(I.getOperand(2).getReg()); 1123 if (I.getNumMemOperands()) 1124 addMemoryOperands(*I.memoperands_begin(), MIB); 1125 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1126 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1127 Result &= BuildCOPY(ResVReg, MIB->getOperand(0).getReg(), I); 1128 return Result; 1129 } 1130 1131 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1132 const SPIRVType *ResType, 1133 MachineInstr &I, 1134 unsigned NewOpcode, 1135 unsigned NegateOpcode) const { 1136 bool Result = true; 1137 assert(I.hasOneMemOperand()); 1138 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1139 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1140 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1141 auto ScopeConstant = buildI32Constant(Scope, I); 1142 Register ScopeReg = ScopeConstant.first; 1143 Result &= ScopeConstant.second; 1144 1145 Register Ptr = I.getOperand(1).getReg(); 1146 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1147 // auto ScSem = 1148 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1149 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1150 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1151 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1152 Register MemSemReg = MemSemConstant.first; 1153 Result &= MemSemConstant.second; 1154 1155 Register ValueReg = I.getOperand(2).getReg(); 1156 if (NegateOpcode != 0) { 1157 // Translation with negative value operand is requested 1158 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1159 Result &= selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode); 1160 ValueReg = TmpReg; 1161 } 1162 1163 return Result && 1164 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1165 .addDef(ResVReg) 1166 .addUse(GR.getSPIRVTypeID(ResType)) 1167 .addUse(Ptr) 1168 .addUse(ScopeReg) 1169 .addUse(MemSemReg) 1170 .addUse(ValueReg) 1171 .constrainAllUses(TII, TRI, RBI); 1172 } 1173 1174 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1175 unsigned ArgI = I.getNumOperands() - 1; 1176 Register SrcReg = 1177 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1178 SPIRVType *DefType = 1179 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1180 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1181 report_fatal_error( 1182 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1183 1184 SPIRVType *ScalarType = 1185 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1186 MachineBasicBlock &BB = *I.getParent(); 1187 bool Res = false; 1188 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1189 Register ResVReg = I.getOperand(i).getReg(); 1190 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1191 if (!ResType) { 1192 // There was no "assign type" actions, let's fix this now 1193 ResType = ScalarType; 1194 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1195 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1196 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1197 } 1198 auto MIB = 1199 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1200 .addDef(ResVReg) 1201 .addUse(GR.getSPIRVTypeID(ResType)) 1202 .addUse(SrcReg) 1203 .addImm(static_cast<int64_t>(i)); 1204 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1205 } 1206 return Res; 1207 } 1208 1209 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1210 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1211 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1212 auto MemSemConstant = buildI32Constant(MemSem, I); 1213 Register MemSemReg = MemSemConstant.first; 1214 bool Result = MemSemConstant.second; 1215 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1216 uint32_t Scope = static_cast<uint32_t>( 1217 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1218 auto ScopeConstant = buildI32Constant(Scope, I); 1219 Register ScopeReg = ScopeConstant.first; 1220 Result &= ScopeConstant.second; 1221 MachineBasicBlock &BB = *I.getParent(); 1222 return Result && 1223 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1224 .addUse(ScopeReg) 1225 .addUse(MemSemReg) 1226 .constrainAllUses(TII, TRI, RBI); 1227 } 1228 1229 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1230 const SPIRVType *ResType, 1231 MachineInstr &I, 1232 unsigned Opcode) const { 1233 Type *ResTy = nullptr; 1234 StringRef ResName; 1235 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1236 report_fatal_error( 1237 "Not enough info to select the arithmetic with overflow instruction"); 1238 if (!ResTy || !ResTy->isStructTy()) 1239 report_fatal_error("Expect struct type result for the arithmetic " 1240 "with overflow instruction"); 1241 // "Result Type must be from OpTypeStruct. The struct must have two members, 1242 // and the two members must be the same type." 1243 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1244 ResTy = StructType::get(ResElemTy, ResElemTy); 1245 // Build SPIR-V types and constant(s) if needed. 1246 MachineIRBuilder MIRBuilder(I); 1247 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1248 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1249 assert(I.getNumDefs() > 1 && "Not enought operands"); 1250 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1251 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1252 if (N > 1) 1253 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1254 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1255 Register ZeroReg = buildZerosVal(ResType, I); 1256 // A new virtual register to store the result struct. 1257 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1258 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1259 // Build the result name if needed. 1260 if (ResName.size() > 0) 1261 buildOpName(StructVReg, ResName, MIRBuilder); 1262 // Build the arithmetic with overflow instruction. 1263 MachineBasicBlock &BB = *I.getParent(); 1264 auto MIB = 1265 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1266 .addDef(StructVReg) 1267 .addUse(GR.getSPIRVTypeID(StructType)); 1268 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1269 MIB.addUse(I.getOperand(i).getReg()); 1270 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1271 // Build instructions to extract fields of the instruction's result. 1272 // A new virtual register to store the higher part of the result struct. 1273 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1274 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1275 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1276 auto MIB = 1277 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1278 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1279 .addUse(GR.getSPIRVTypeID(ResType)) 1280 .addUse(StructVReg) 1281 .addImm(i); 1282 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1283 } 1284 // Build boolean value from the higher part. 1285 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1286 .addDef(I.getOperand(1).getReg()) 1287 .addUse(BoolTypeReg) 1288 .addUse(HigherVReg) 1289 .addUse(ZeroReg) 1290 .constrainAllUses(TII, TRI, RBI); 1291 } 1292 1293 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1294 const SPIRVType *ResType, 1295 MachineInstr &I) const { 1296 bool Result = true; 1297 Register ScopeReg; 1298 Register MemSemEqReg; 1299 Register MemSemNeqReg; 1300 Register Ptr = I.getOperand(2).getReg(); 1301 if (!isa<GIntrinsic>(I)) { 1302 assert(I.hasOneMemOperand()); 1303 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1304 unsigned Scope = static_cast<uint32_t>(getMemScope( 1305 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1306 auto ScopeConstant = buildI32Constant(Scope, I); 1307 ScopeReg = ScopeConstant.first; 1308 Result &= ScopeConstant.second; 1309 1310 unsigned ScSem = static_cast<uint32_t>( 1311 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1312 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1313 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1314 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1315 MemSemEqReg = MemSemEqConstant.first; 1316 Result &= MemSemEqConstant.second; 1317 AtomicOrdering FO = MemOp->getFailureOrdering(); 1318 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1319 if (MemSemEq == MemSemNeq) 1320 MemSemNeqReg = MemSemEqReg; 1321 else { 1322 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1323 MemSemNeqReg = MemSemNeqConstant.first; 1324 Result &= MemSemNeqConstant.second; 1325 } 1326 } else { 1327 ScopeReg = I.getOperand(5).getReg(); 1328 MemSemEqReg = I.getOperand(6).getReg(); 1329 MemSemNeqReg = I.getOperand(7).getReg(); 1330 } 1331 1332 Register Cmp = I.getOperand(3).getReg(); 1333 Register Val = I.getOperand(4).getReg(); 1334 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1335 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1336 const DebugLoc &DL = I.getDebugLoc(); 1337 Result &= 1338 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1339 .addDef(ACmpRes) 1340 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1341 .addUse(Ptr) 1342 .addUse(ScopeReg) 1343 .addUse(MemSemEqReg) 1344 .addUse(MemSemNeqReg) 1345 .addUse(Val) 1346 .addUse(Cmp) 1347 .constrainAllUses(TII, TRI, RBI); 1348 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1349 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1350 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1351 .addDef(CmpSuccReg) 1352 .addUse(GR.getSPIRVTypeID(BoolTy)) 1353 .addUse(ACmpRes) 1354 .addUse(Cmp) 1355 .constrainAllUses(TII, TRI, RBI); 1356 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1357 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1358 .addDef(TmpReg) 1359 .addUse(GR.getSPIRVTypeID(ResType)) 1360 .addUse(ACmpRes) 1361 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1362 .addImm(0) 1363 .constrainAllUses(TII, TRI, RBI); 1364 return Result && 1365 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1366 .addDef(ResVReg) 1367 .addUse(GR.getSPIRVTypeID(ResType)) 1368 .addUse(CmpSuccReg) 1369 .addUse(TmpReg) 1370 .addImm(1) 1371 .constrainAllUses(TII, TRI, RBI); 1372 } 1373 1374 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1375 switch (SC) { 1376 case SPIRV::StorageClass::Workgroup: 1377 case SPIRV::StorageClass::CrossWorkgroup: 1378 case SPIRV::StorageClass::Function: 1379 return true; 1380 default: 1381 return false; 1382 } 1383 } 1384 1385 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1386 switch (SC) { 1387 case SPIRV::StorageClass::DeviceOnlyINTEL: 1388 case SPIRV::StorageClass::HostOnlyINTEL: 1389 return true; 1390 default: 1391 return false; 1392 } 1393 } 1394 1395 // Returns true ResVReg is referred only from global vars and OpName's. 1396 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1397 bool IsGRef = false; 1398 bool IsAllowedRefs = 1399 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1400 [&IsGRef](auto const &It) { 1401 unsigned Opcode = It.getOpcode(); 1402 if (Opcode == SPIRV::OpConstantComposite || 1403 Opcode == SPIRV::OpVariable || 1404 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1405 return IsGRef = true; 1406 return Opcode == SPIRV::OpName; 1407 }); 1408 return IsAllowedRefs && IsGRef; 1409 } 1410 1411 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1412 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1413 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1414 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1415 } 1416 1417 MachineInstrBuilder 1418 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1419 Register Src, Register DestType, 1420 uint32_t Opcode) const { 1421 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1422 TII.get(SPIRV::OpSpecConstantOp)) 1423 .addDef(Dest) 1424 .addUse(DestType) 1425 .addImm(Opcode) 1426 .addUse(Src); 1427 } 1428 1429 MachineInstrBuilder 1430 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1431 SPIRVType *SrcPtrTy) const { 1432 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1433 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1434 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1435 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1436 SPIRV::StorageClass::Generic), 1437 GR.getPointerSize())); 1438 MachineFunction *MF = I.getParent()->getParent(); 1439 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1440 MachineInstrBuilder MIB = buildSpecConstantOp( 1441 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1442 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1443 GR.add(MIB.getInstr(), MF, Tmp); 1444 return MIB; 1445 } 1446 1447 // In SPIR-V address space casting can only happen to and from the Generic 1448 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1449 // pointers to and from Generic pointers. As such, we can convert e.g. from 1450 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1451 // other combinations can only be done by a bitcast, and are probably not safe. 1452 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1453 const SPIRVType *ResType, 1454 MachineInstr &I) const { 1455 MachineBasicBlock &BB = *I.getParent(); 1456 const DebugLoc &DL = I.getDebugLoc(); 1457 1458 Register SrcPtr = I.getOperand(1).getReg(); 1459 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1460 1461 // don't generate a cast for a null that may be represented by OpTypeInt 1462 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1463 ResType->getOpcode() != SPIRV::OpTypePointer) 1464 return BuildCOPY(ResVReg, SrcPtr, I); 1465 1466 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1467 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1468 1469 if (isASCastInGVar(MRI, ResVReg)) { 1470 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1471 // are expressed by OpSpecConstantOp with an Opcode. 1472 // TODO: maybe insert a check whether the Kernel capability was declared and 1473 // so PtrCastToGeneric/GenericCastToPtr are available. 1474 unsigned SpecOpcode = 1475 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1476 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1477 : (SrcSC == SPIRV::StorageClass::Generic && 1478 isGenericCastablePtr(DstSC) 1479 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1480 : 0); 1481 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1482 // correct value of ResType and use general i8* instead. Maybe this should 1483 // be addressed in the emit-intrinsic step to infer a correct 1484 // OpConstantComposite type. 1485 if (SpecOpcode) { 1486 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1487 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1488 .constrainAllUses(TII, TRI, RBI); 1489 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1490 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1491 return MIB.constrainAllUses(TII, TRI, RBI) && 1492 buildSpecConstantOp( 1493 I, ResVReg, MIB->getOperand(0).getReg(), 1494 getUcharPtrTypeReg(I, DstSC), 1495 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1496 .constrainAllUses(TII, TRI, RBI); 1497 } 1498 } 1499 1500 // don't generate a cast between identical storage classes 1501 if (SrcSC == DstSC) 1502 return BuildCOPY(ResVReg, SrcPtr, I); 1503 1504 if ((SrcSC == SPIRV::StorageClass::Function && 1505 DstSC == SPIRV::StorageClass::Private) || 1506 (DstSC == SPIRV::StorageClass::Function && 1507 SrcSC == SPIRV::StorageClass::Private)) 1508 return BuildCOPY(ResVReg, SrcPtr, I); 1509 1510 // Casting from an eligible pointer to Generic. 1511 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1512 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1513 // Casting from Generic to an eligible pointer. 1514 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1515 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1516 // Casting between 2 eligible pointers using Generic as an intermediary. 1517 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1518 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1519 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1520 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1521 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1522 .addDef(Tmp) 1523 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1524 .addUse(SrcPtr) 1525 .constrainAllUses(TII, TRI, RBI); 1526 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1527 .addDef(ResVReg) 1528 .addUse(GR.getSPIRVTypeID(ResType)) 1529 .addUse(Tmp) 1530 .constrainAllUses(TII, TRI, RBI); 1531 } 1532 1533 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1534 // be applied 1535 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1536 return selectUnOp(ResVReg, ResType, I, 1537 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1538 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1539 return selectUnOp(ResVReg, ResType, I, 1540 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1541 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1542 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1543 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1544 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1545 1546 // Bitcast for pointers requires that the address spaces must match 1547 return false; 1548 } 1549 1550 static unsigned getFCmpOpcode(unsigned PredNum) { 1551 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1552 switch (Pred) { 1553 case CmpInst::FCMP_OEQ: 1554 return SPIRV::OpFOrdEqual; 1555 case CmpInst::FCMP_OGE: 1556 return SPIRV::OpFOrdGreaterThanEqual; 1557 case CmpInst::FCMP_OGT: 1558 return SPIRV::OpFOrdGreaterThan; 1559 case CmpInst::FCMP_OLE: 1560 return SPIRV::OpFOrdLessThanEqual; 1561 case CmpInst::FCMP_OLT: 1562 return SPIRV::OpFOrdLessThan; 1563 case CmpInst::FCMP_ONE: 1564 return SPIRV::OpFOrdNotEqual; 1565 case CmpInst::FCMP_ORD: 1566 return SPIRV::OpOrdered; 1567 case CmpInst::FCMP_UEQ: 1568 return SPIRV::OpFUnordEqual; 1569 case CmpInst::FCMP_UGE: 1570 return SPIRV::OpFUnordGreaterThanEqual; 1571 case CmpInst::FCMP_UGT: 1572 return SPIRV::OpFUnordGreaterThan; 1573 case CmpInst::FCMP_ULE: 1574 return SPIRV::OpFUnordLessThanEqual; 1575 case CmpInst::FCMP_ULT: 1576 return SPIRV::OpFUnordLessThan; 1577 case CmpInst::FCMP_UNE: 1578 return SPIRV::OpFUnordNotEqual; 1579 case CmpInst::FCMP_UNO: 1580 return SPIRV::OpUnordered; 1581 default: 1582 llvm_unreachable("Unknown predicate type for FCmp"); 1583 } 1584 } 1585 1586 static unsigned getICmpOpcode(unsigned PredNum) { 1587 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1588 switch (Pred) { 1589 case CmpInst::ICMP_EQ: 1590 return SPIRV::OpIEqual; 1591 case CmpInst::ICMP_NE: 1592 return SPIRV::OpINotEqual; 1593 case CmpInst::ICMP_SGE: 1594 return SPIRV::OpSGreaterThanEqual; 1595 case CmpInst::ICMP_SGT: 1596 return SPIRV::OpSGreaterThan; 1597 case CmpInst::ICMP_SLE: 1598 return SPIRV::OpSLessThanEqual; 1599 case CmpInst::ICMP_SLT: 1600 return SPIRV::OpSLessThan; 1601 case CmpInst::ICMP_UGE: 1602 return SPIRV::OpUGreaterThanEqual; 1603 case CmpInst::ICMP_UGT: 1604 return SPIRV::OpUGreaterThan; 1605 case CmpInst::ICMP_ULE: 1606 return SPIRV::OpULessThanEqual; 1607 case CmpInst::ICMP_ULT: 1608 return SPIRV::OpULessThan; 1609 default: 1610 llvm_unreachable("Unknown predicate type for ICmp"); 1611 } 1612 } 1613 1614 static unsigned getPtrCmpOpcode(unsigned Pred) { 1615 switch (static_cast<CmpInst::Predicate>(Pred)) { 1616 case CmpInst::ICMP_EQ: 1617 return SPIRV::OpPtrEqual; 1618 case CmpInst::ICMP_NE: 1619 return SPIRV::OpPtrNotEqual; 1620 default: 1621 llvm_unreachable("Unknown predicate type for pointer comparison"); 1622 } 1623 } 1624 1625 // Return the logical operation, or abort if none exists. 1626 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1627 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1628 switch (Pred) { 1629 case CmpInst::ICMP_EQ: 1630 return SPIRV::OpLogicalEqual; 1631 case CmpInst::ICMP_NE: 1632 return SPIRV::OpLogicalNotEqual; 1633 default: 1634 llvm_unreachable("Unknown predicate type for Bool comparison"); 1635 } 1636 } 1637 1638 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1639 if (!LLVMFloatTy) 1640 return APFloat::getZero(APFloat::IEEEsingle()); 1641 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1642 case Type::HalfTyID: 1643 return APFloat::getZero(APFloat::IEEEhalf()); 1644 default: 1645 case Type::FloatTyID: 1646 return APFloat::getZero(APFloat::IEEEsingle()); 1647 case Type::DoubleTyID: 1648 return APFloat::getZero(APFloat::IEEEdouble()); 1649 } 1650 } 1651 1652 static APFloat getOneFP(const Type *LLVMFloatTy) { 1653 if (!LLVMFloatTy) 1654 return APFloat::getOne(APFloat::IEEEsingle()); 1655 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1656 case Type::HalfTyID: 1657 return APFloat::getOne(APFloat::IEEEhalf()); 1658 default: 1659 case Type::FloatTyID: 1660 return APFloat::getOne(APFloat::IEEEsingle()); 1661 case Type::DoubleTyID: 1662 return APFloat::getOne(APFloat::IEEEdouble()); 1663 } 1664 } 1665 1666 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1667 const SPIRVType *ResType, 1668 MachineInstr &I, 1669 unsigned OpAnyOrAll) const { 1670 assert(I.getNumOperands() == 3); 1671 assert(I.getOperand(2).isReg()); 1672 MachineBasicBlock &BB = *I.getParent(); 1673 Register InputRegister = I.getOperand(2).getReg(); 1674 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1675 1676 if (!InputType) 1677 report_fatal_error("Input Type could not be determined."); 1678 1679 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1680 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1681 if (IsBoolTy && !IsVectorTy) { 1682 assert(ResVReg == I.getOperand(0).getReg()); 1683 return BuildCOPY(ResVReg, InputRegister, I); 1684 } 1685 1686 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1687 unsigned SpirvNotEqualId = 1688 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1689 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1690 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1691 Register NotEqualReg = ResVReg; 1692 1693 if (IsVectorTy) { 1694 NotEqualReg = IsBoolTy ? InputRegister 1695 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1696 const unsigned NumElts = InputType->getOperand(2).getImm(); 1697 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1698 } 1699 1700 bool Result = true; 1701 if (!IsBoolTy) { 1702 Register ConstZeroReg = 1703 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1704 1705 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1706 .addDef(NotEqualReg) 1707 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1708 .addUse(InputRegister) 1709 .addUse(ConstZeroReg) 1710 .constrainAllUses(TII, TRI, RBI); 1711 } 1712 1713 if (!IsVectorTy) 1714 return Result; 1715 1716 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1717 .addDef(ResVReg) 1718 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1719 .addUse(NotEqualReg) 1720 .constrainAllUses(TII, TRI, RBI); 1721 } 1722 1723 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1724 const SPIRVType *ResType, 1725 MachineInstr &I) const { 1726 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1727 } 1728 1729 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1730 const SPIRVType *ResType, 1731 MachineInstr &I) const { 1732 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1733 } 1734 1735 // Select the OpDot instruction for the given float dot 1736 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1737 const SPIRVType *ResType, 1738 MachineInstr &I) const { 1739 assert(I.getNumOperands() == 4); 1740 assert(I.getOperand(2).isReg()); 1741 assert(I.getOperand(3).isReg()); 1742 1743 [[maybe_unused]] SPIRVType *VecType = 1744 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1745 1746 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1747 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1748 "dot product requires a vector of at least 2 components"); 1749 1750 [[maybe_unused]] SPIRVType *EltType = 1751 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1752 1753 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1754 1755 MachineBasicBlock &BB = *I.getParent(); 1756 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1757 .addDef(ResVReg) 1758 .addUse(GR.getSPIRVTypeID(ResType)) 1759 .addUse(I.getOperand(2).getReg()) 1760 .addUse(I.getOperand(3).getReg()) 1761 .constrainAllUses(TII, TRI, RBI); 1762 } 1763 1764 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1765 const SPIRVType *ResType, 1766 MachineInstr &I, 1767 bool Signed) const { 1768 assert(I.getNumOperands() == 4); 1769 assert(I.getOperand(2).isReg()); 1770 assert(I.getOperand(3).isReg()); 1771 MachineBasicBlock &BB = *I.getParent(); 1772 1773 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1774 return BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1775 .addDef(ResVReg) 1776 .addUse(GR.getSPIRVTypeID(ResType)) 1777 .addUse(I.getOperand(2).getReg()) 1778 .addUse(I.getOperand(3).getReg()) 1779 .constrainAllUses(TII, TRI, RBI); 1780 } 1781 1782 // Since pre-1.6 SPIRV has no integer dot implementation, 1783 // expand by piecewise multiplying and adding the results 1784 bool SPIRVInstructionSelector::selectIntegerDotExpansion( 1785 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1786 assert(I.getNumOperands() == 4); 1787 assert(I.getOperand(2).isReg()); 1788 assert(I.getOperand(3).isReg()); 1789 MachineBasicBlock &BB = *I.getParent(); 1790 1791 // Multiply the vectors, then sum the results 1792 Register Vec0 = I.getOperand(2).getReg(); 1793 Register Vec1 = I.getOperand(3).getReg(); 1794 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1795 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1796 1797 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1798 .addDef(TmpVec) 1799 .addUse(GR.getSPIRVTypeID(VecType)) 1800 .addUse(Vec0) 1801 .addUse(Vec1) 1802 .constrainAllUses(TII, TRI, RBI); 1803 1804 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1805 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1806 "dot product requires a vector of at least 2 components"); 1807 1808 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1809 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1810 .addDef(Res) 1811 .addUse(GR.getSPIRVTypeID(ResType)) 1812 .addUse(TmpVec) 1813 .addImm(0) 1814 .constrainAllUses(TII, TRI, RBI); 1815 1816 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1817 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1818 1819 Result &= 1820 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1821 .addDef(Elt) 1822 .addUse(GR.getSPIRVTypeID(ResType)) 1823 .addUse(TmpVec) 1824 .addImm(i) 1825 .constrainAllUses(TII, TRI, RBI); 1826 1827 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1828 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1829 : ResVReg; 1830 1831 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1832 .addDef(Sum) 1833 .addUse(GR.getSPIRVTypeID(ResType)) 1834 .addUse(Res) 1835 .addUse(Elt) 1836 .constrainAllUses(TII, TRI, RBI); 1837 Res = Sum; 1838 } 1839 1840 return Result; 1841 } 1842 1843 template <bool Signed> 1844 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1845 const SPIRVType *ResType, 1846 MachineInstr &I) const { 1847 assert(I.getNumOperands() == 5); 1848 assert(I.getOperand(2).isReg()); 1849 assert(I.getOperand(3).isReg()); 1850 assert(I.getOperand(4).isReg()); 1851 MachineBasicBlock &BB = *I.getParent(); 1852 1853 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1854 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1855 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1856 .addDef(Dot) 1857 .addUse(GR.getSPIRVTypeID(ResType)) 1858 .addUse(I.getOperand(2).getReg()) 1859 .addUse(I.getOperand(3).getReg()) 1860 .constrainAllUses(TII, TRI, RBI); 1861 1862 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1863 .addDef(ResVReg) 1864 .addUse(GR.getSPIRVTypeID(ResType)) 1865 .addUse(Dot) 1866 .addUse(I.getOperand(4).getReg()) 1867 .constrainAllUses(TII, TRI, RBI); 1868 } 1869 1870 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1871 // extract the elements of the packed inputs, multiply them and add the result 1872 // to the accumulator. 1873 template <bool Signed> 1874 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1875 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1876 assert(I.getNumOperands() == 5); 1877 assert(I.getOperand(2).isReg()); 1878 assert(I.getOperand(3).isReg()); 1879 assert(I.getOperand(4).isReg()); 1880 MachineBasicBlock &BB = *I.getParent(); 1881 1882 bool Result = true; 1883 1884 // Acc = C 1885 Register Acc = I.getOperand(4).getReg(); 1886 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1887 auto ExtractOp = 1888 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1889 1890 // Extract the i8 element, multiply and add it to the accumulator 1891 for (unsigned i = 0; i < 4; i++) { 1892 // A[i] 1893 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1894 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1895 .addDef(AElt) 1896 .addUse(GR.getSPIRVTypeID(ResType)) 1897 .addUse(I.getOperand(2).getReg()) 1898 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1899 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1900 .constrainAllUses(TII, TRI, RBI); 1901 1902 // B[i] 1903 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1904 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1905 .addDef(BElt) 1906 .addUse(GR.getSPIRVTypeID(ResType)) 1907 .addUse(I.getOperand(3).getReg()) 1908 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1909 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1910 .constrainAllUses(TII, TRI, RBI); 1911 1912 // A[i] * B[i] 1913 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1914 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1915 .addDef(Mul) 1916 .addUse(GR.getSPIRVTypeID(ResType)) 1917 .addUse(AElt) 1918 .addUse(BElt) 1919 .constrainAllUses(TII, TRI, RBI); 1920 1921 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1922 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1923 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1924 .addDef(MaskMul) 1925 .addUse(GR.getSPIRVTypeID(ResType)) 1926 .addUse(Mul) 1927 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1928 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1929 .constrainAllUses(TII, TRI, RBI); 1930 1931 // Acc = Acc + A[i] * B[i] 1932 Register Sum = 1933 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1934 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1935 .addDef(Sum) 1936 .addUse(GR.getSPIRVTypeID(ResType)) 1937 .addUse(Acc) 1938 .addUse(MaskMul) 1939 .constrainAllUses(TII, TRI, RBI); 1940 1941 Acc = Sum; 1942 } 1943 1944 return Result; 1945 } 1946 1947 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1948 /// does not have a saturate builtin. 1949 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1950 const SPIRVType *ResType, 1951 MachineInstr &I) const { 1952 assert(I.getNumOperands() == 3); 1953 assert(I.getOperand(2).isReg()); 1954 MachineBasicBlock &BB = *I.getParent(); 1955 Register VZero = buildZerosValF(ResType, I); 1956 Register VOne = buildOnesValF(ResType, I); 1957 1958 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1959 .addDef(ResVReg) 1960 .addUse(GR.getSPIRVTypeID(ResType)) 1961 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1962 .addImm(GL::FClamp) 1963 .addUse(I.getOperand(2).getReg()) 1964 .addUse(VZero) 1965 .addUse(VOne) 1966 .constrainAllUses(TII, TRI, RBI); 1967 } 1968 1969 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1970 const SPIRVType *ResType, 1971 MachineInstr &I) const { 1972 assert(I.getNumOperands() == 3); 1973 assert(I.getOperand(2).isReg()); 1974 MachineBasicBlock &BB = *I.getParent(); 1975 Register InputRegister = I.getOperand(2).getReg(); 1976 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1977 auto &DL = I.getDebugLoc(); 1978 1979 if (!InputType) 1980 report_fatal_error("Input Type could not be determined."); 1981 1982 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1983 1984 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1985 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1986 1987 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1988 1989 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1990 Register SignReg = NeedsConversion 1991 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1992 : ResVReg; 1993 1994 bool Result = 1995 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1996 .addDef(SignReg) 1997 .addUse(GR.getSPIRVTypeID(InputType)) 1998 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1999 .addImm(SignOpcode) 2000 .addUse(InputRegister) 2001 .constrainAllUses(TII, TRI, RBI); 2002 2003 if (NeedsConversion) { 2004 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 2005 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 2006 .addDef(ResVReg) 2007 .addUse(GR.getSPIRVTypeID(ResType)) 2008 .addUse(SignReg) 2009 .constrainAllUses(TII, TRI, RBI); 2010 } 2011 2012 return Result; 2013 } 2014 2015 bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, 2016 const SPIRVType *ResType, 2017 MachineInstr &I, 2018 unsigned Opcode) const { 2019 MachineBasicBlock &BB = *I.getParent(); 2020 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2021 2022 auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2023 .addDef(ResVReg) 2024 .addUse(GR.getSPIRVTypeID(ResType)) 2025 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, 2026 IntTy, TII)); 2027 2028 for (unsigned J = 2; J < I.getNumOperands(); J++) { 2029 BMI.addUse(I.getOperand(J).getReg()); 2030 } 2031 2032 return BMI.constrainAllUses(TII, TRI, RBI); 2033 } 2034 2035 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 2036 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2037 2038 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2039 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 2040 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 2041 bool Result = selectWaveOpInst(BallotReg, BallotType, I, 2042 SPIRV::OpGroupNonUniformBallot); 2043 2044 MachineBasicBlock &BB = *I.getParent(); 2045 Result &= 2046 BuildMI(BB, I, I.getDebugLoc(), 2047 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 2048 .addDef(ResVReg) 2049 .addUse(GR.getSPIRVTypeID(ResType)) 2050 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 2051 .addImm(SPIRV::GroupOperation::Reduce) 2052 .addUse(BallotReg) 2053 .constrainAllUses(TII, TRI, RBI); 2054 2055 return Result; 2056 } 2057 2058 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2059 const SPIRVType *ResType, 2060 MachineInstr &I) const { 2061 MachineBasicBlock &BB = *I.getParent(); 2062 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2063 .addDef(ResVReg) 2064 .addUse(GR.getSPIRVTypeID(ResType)) 2065 .addUse(I.getOperand(1).getReg()) 2066 .constrainAllUses(TII, TRI, RBI); 2067 } 2068 2069 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2070 const SPIRVType *ResType, 2071 MachineInstr &I) const { 2072 // There is no way to implement `freeze` correctly without support on SPIR-V 2073 // standard side, but we may at least address a simple (static) case when 2074 // undef/poison value presence is obvious. The main benefit of even 2075 // incomplete `freeze` support is preventing of translation from crashing due 2076 // to lack of support on legalization and instruction selection steps. 2077 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2078 return false; 2079 Register OpReg = I.getOperand(1).getReg(); 2080 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2081 Register Reg; 2082 switch (Def->getOpcode()) { 2083 case SPIRV::ASSIGN_TYPE: 2084 if (MachineInstr *AssignToDef = 2085 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2086 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2087 Reg = Def->getOperand(2).getReg(); 2088 } 2089 break; 2090 case SPIRV::OpUndef: 2091 Reg = Def->getOperand(1).getReg(); 2092 break; 2093 } 2094 unsigned DestOpCode; 2095 if (Reg.isValid()) { 2096 DestOpCode = SPIRV::OpConstantNull; 2097 } else { 2098 DestOpCode = TargetOpcode::COPY; 2099 Reg = OpReg; 2100 } 2101 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2102 .addDef(I.getOperand(0).getReg()) 2103 .addUse(Reg) 2104 .constrainAllUses(TII, TRI, RBI); 2105 } 2106 return false; 2107 } 2108 2109 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2110 const SPIRVType *ResType) { 2111 Register OpReg = ResType->getOperand(2).getReg(); 2112 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2113 if (!OpDef) 2114 return 0; 2115 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2116 OpDef->getOperand(1).isReg()) { 2117 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2118 OpDef = RefDef; 2119 } 2120 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2121 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2122 : 0; 2123 return N; 2124 } 2125 2126 // Return true if the type represents a constant register 2127 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2128 SmallPtrSet<SPIRVType *, 4> &Visited) { 2129 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2130 OpDef->getOperand(1).isReg()) { 2131 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2132 OpDef = RefDef; 2133 } 2134 2135 if (Visited.contains(OpDef)) 2136 return true; 2137 Visited.insert(OpDef); 2138 2139 unsigned Opcode = OpDef->getOpcode(); 2140 switch (Opcode) { 2141 case TargetOpcode::G_CONSTANT: 2142 case TargetOpcode::G_FCONSTANT: 2143 return true; 2144 case TargetOpcode::G_INTRINSIC: 2145 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2146 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2147 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2148 Intrinsic::spv_const_composite; 2149 case TargetOpcode::G_BUILD_VECTOR: 2150 case TargetOpcode::G_SPLAT_VECTOR: { 2151 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2152 i++) { 2153 SPIRVType *OpNestedDef = 2154 OpDef->getOperand(i).isReg() 2155 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2156 : nullptr; 2157 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2158 return false; 2159 } 2160 return true; 2161 } 2162 } 2163 return false; 2164 } 2165 2166 // Return true if the virtual register represents a constant 2167 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2168 SmallPtrSet<SPIRVType *, 4> Visited; 2169 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2170 return isConstReg(MRI, OpDef, Visited); 2171 return false; 2172 } 2173 2174 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2175 const SPIRVType *ResType, 2176 MachineInstr &I) const { 2177 unsigned N = 0; 2178 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2179 N = GR.getScalarOrVectorComponentCount(ResType); 2180 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2181 N = getArrayComponentCount(MRI, ResType); 2182 else 2183 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2184 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2185 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2186 2187 // check if we may construct a constant vector 2188 bool IsConst = true; 2189 for (unsigned i = I.getNumExplicitDefs(); 2190 i < I.getNumExplicitOperands() && IsConst; ++i) 2191 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2192 IsConst = false; 2193 2194 if (!IsConst && N < 2) 2195 report_fatal_error( 2196 "There must be at least two constituent operands in a vector"); 2197 2198 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2199 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2200 TII.get(IsConst ? SPIRV::OpConstantComposite 2201 : SPIRV::OpCompositeConstruct)) 2202 .addDef(ResVReg) 2203 .addUse(GR.getSPIRVTypeID(ResType)); 2204 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2205 MIB.addUse(I.getOperand(i).getReg()); 2206 return MIB.constrainAllUses(TII, TRI, RBI); 2207 } 2208 2209 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2210 const SPIRVType *ResType, 2211 MachineInstr &I) const { 2212 unsigned N = 0; 2213 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2214 N = GR.getScalarOrVectorComponentCount(ResType); 2215 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2216 N = getArrayComponentCount(MRI, ResType); 2217 else 2218 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2219 2220 unsigned OpIdx = I.getNumExplicitDefs(); 2221 if (!I.getOperand(OpIdx).isReg()) 2222 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2223 2224 // check if we may construct a constant vector 2225 Register OpReg = I.getOperand(OpIdx).getReg(); 2226 bool IsConst = isConstReg(MRI, OpReg); 2227 2228 if (!IsConst && N < 2) 2229 report_fatal_error( 2230 "There must be at least two constituent operands in a vector"); 2231 2232 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2233 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2234 TII.get(IsConst ? SPIRV::OpConstantComposite 2235 : SPIRV::OpCompositeConstruct)) 2236 .addDef(ResVReg) 2237 .addUse(GR.getSPIRVTypeID(ResType)); 2238 for (unsigned i = 0; i < N; ++i) 2239 MIB.addUse(OpReg); 2240 return MIB.constrainAllUses(TII, TRI, RBI); 2241 } 2242 2243 bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, 2244 const SPIRVType *ResType, 2245 MachineInstr &I) const { 2246 2247 unsigned Opcode; 2248 2249 if (STI.canUseExtension( 2250 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || 2251 STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { 2252 Opcode = SPIRV::OpDemoteToHelperInvocation; 2253 } else { 2254 Opcode = SPIRV::OpKill; 2255 // OpKill must be the last operation of any basic block. 2256 MachineInstr *NextI = I.getNextNode(); 2257 NextI->removeFromParent(); 2258 } 2259 2260 MachineBasicBlock &BB = *I.getParent(); 2261 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2262 .constrainAllUses(TII, TRI, RBI); 2263 } 2264 2265 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2266 const SPIRVType *ResType, 2267 unsigned CmpOpc, 2268 MachineInstr &I) const { 2269 Register Cmp0 = I.getOperand(2).getReg(); 2270 Register Cmp1 = I.getOperand(3).getReg(); 2271 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2272 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2273 "CMP operands should have the same type"); 2274 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2275 .addDef(ResVReg) 2276 .addUse(GR.getSPIRVTypeID(ResType)) 2277 .addUse(Cmp0) 2278 .addUse(Cmp1) 2279 .constrainAllUses(TII, TRI, RBI); 2280 } 2281 2282 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2283 const SPIRVType *ResType, 2284 MachineInstr &I) const { 2285 auto Pred = I.getOperand(1).getPredicate(); 2286 unsigned CmpOpc; 2287 2288 Register CmpOperand = I.getOperand(2).getReg(); 2289 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2290 CmpOpc = getPtrCmpOpcode(Pred); 2291 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2292 CmpOpc = getBoolCmpOpcode(Pred); 2293 else 2294 CmpOpc = getICmpOpcode(Pred); 2295 return selectCmp(ResVReg, ResType, CmpOpc, I); 2296 } 2297 2298 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2299 const MachineInstr &I, 2300 int OpIdx) const { 2301 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2302 "Expected G_FCONSTANT"); 2303 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2304 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2305 } 2306 2307 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2308 const MachineInstr &I, 2309 int OpIdx) const { 2310 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2311 "Expected G_CONSTANT"); 2312 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2313 } 2314 2315 std::pair<Register, bool> 2316 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2317 const SPIRVType *ResType) const { 2318 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2319 const SPIRVType *SpvI32Ty = 2320 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2321 // Find a constant in DT or build a new one. 2322 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2323 Register NewReg = GR.find(ConstInt, GR.CurMF); 2324 bool Result = true; 2325 if (!NewReg.isValid()) { 2326 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2327 GR.add(ConstInt, GR.CurMF, NewReg); 2328 MachineInstr *MI; 2329 MachineBasicBlock &BB = *I.getParent(); 2330 if (Val == 0) { 2331 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2332 .addDef(NewReg) 2333 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2334 } else { 2335 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2336 .addDef(NewReg) 2337 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2338 .addImm(APInt(32, Val).getZExtValue()); 2339 } 2340 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2341 } 2342 return {NewReg, Result}; 2343 } 2344 2345 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2346 const SPIRVType *ResType, 2347 MachineInstr &I) const { 2348 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2349 return selectCmp(ResVReg, ResType, CmpOp, I); 2350 } 2351 2352 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2353 MachineInstr &I) const { 2354 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2355 bool ZeroAsNull = STI.isOpenCLEnv(); 2356 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2357 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2358 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2359 } 2360 2361 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2362 MachineInstr &I) const { 2363 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2364 bool ZeroAsNull = STI.isOpenCLEnv(); 2365 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2366 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2367 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2368 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2369 } 2370 2371 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2372 MachineInstr &I) const { 2373 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2374 bool ZeroAsNull = STI.isOpenCLEnv(); 2375 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2376 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2377 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2378 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2379 } 2380 2381 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2382 const SPIRVType *ResType, 2383 MachineInstr &I) const { 2384 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2385 APInt One = 2386 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2387 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2388 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2389 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2390 } 2391 2392 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2393 const SPIRVType *ResType, 2394 MachineInstr &I, 2395 bool IsSigned) const { 2396 // To extend a bool, we need to use OpSelect between constants. 2397 Register ZeroReg = buildZerosVal(ResType, I); 2398 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2399 bool IsScalarBool = 2400 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2401 unsigned Opcode = 2402 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; 2403 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2404 .addDef(ResVReg) 2405 .addUse(GR.getSPIRVTypeID(ResType)) 2406 .addUse(I.getOperand(1).getReg()) 2407 .addUse(OneReg) 2408 .addUse(ZeroReg) 2409 .constrainAllUses(TII, TRI, RBI); 2410 } 2411 2412 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2413 const SPIRVType *ResType, 2414 MachineInstr &I, bool IsSigned, 2415 unsigned Opcode) const { 2416 Register SrcReg = I.getOperand(1).getReg(); 2417 // We can convert bool value directly to float type without OpConvert*ToF, 2418 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2419 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2420 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2421 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2422 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2423 const unsigned NumElts = ResType->getOperand(2).getImm(); 2424 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2425 } 2426 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2427 selectSelect(SrcReg, TmpType, I, false); 2428 } 2429 return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); 2430 } 2431 2432 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2433 const SPIRVType *ResType, 2434 MachineInstr &I, bool IsSigned) const { 2435 Register SrcReg = I.getOperand(1).getReg(); 2436 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2437 return selectSelect(ResVReg, ResType, I, IsSigned); 2438 2439 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2440 if (SrcType == ResType) 2441 return BuildCOPY(ResVReg, SrcReg, I); 2442 2443 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2444 return selectUnOp(ResVReg, ResType, I, Opcode); 2445 } 2446 2447 bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, 2448 const SPIRVType *ResType, 2449 MachineInstr &I, 2450 bool IsSigned) const { 2451 MachineIRBuilder MIRBuilder(I); 2452 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2453 MachineBasicBlock &BB = *I.getParent(); 2454 // Ensure we have bool. 2455 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2456 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 2457 if (N > 1) 2458 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 2459 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 2460 // Build less-than-equal and less-than. 2461 // TODO: replace with one-liner createVirtualRegister() from 2462 // llvm/lib/Target/SPIRV/SPIRVUtils.cpp when PR #116609 is merged. 2463 Register IsLessEqReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2464 MRI->setType(IsLessEqReg, LLT::scalar(64)); 2465 GR.assignSPIRVTypeToVReg(ResType, IsLessEqReg, MIRBuilder.getMF()); 2466 bool Result = BuildMI(BB, I, I.getDebugLoc(), 2467 TII.get(IsSigned ? SPIRV::OpSLessThanEqual 2468 : SPIRV::OpULessThanEqual)) 2469 .addDef(IsLessEqReg) 2470 .addUse(BoolTypeReg) 2471 .addUse(I.getOperand(1).getReg()) 2472 .addUse(I.getOperand(2).getReg()) 2473 .constrainAllUses(TII, TRI, RBI); 2474 Register IsLessReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2475 MRI->setType(IsLessReg, LLT::scalar(64)); 2476 GR.assignSPIRVTypeToVReg(ResType, IsLessReg, MIRBuilder.getMF()); 2477 Result &= BuildMI(BB, I, I.getDebugLoc(), 2478 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) 2479 .addDef(IsLessReg) 2480 .addUse(BoolTypeReg) 2481 .addUse(I.getOperand(1).getReg()) 2482 .addUse(I.getOperand(2).getReg()) 2483 .constrainAllUses(TII, TRI, RBI); 2484 // Build selects. 2485 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 2486 Register NegOneOrZeroReg = 2487 MRI->createVirtualRegister(GR.getRegClass(ResType)); 2488 MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); 2489 GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); 2490 unsigned SelectOpcode = 2491 N > 1 ? SPIRV::OpSelectVIVCond : SPIRV::OpSelectSISCond; 2492 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2493 .addDef(NegOneOrZeroReg) 2494 .addUse(ResTypeReg) 2495 .addUse(IsLessReg) 2496 .addUse(buildOnesVal(true, ResType, I)) // -1 2497 .addUse(buildZerosVal(ResType, I)) 2498 .constrainAllUses(TII, TRI, RBI); 2499 return Result & BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) 2500 .addDef(ResVReg) 2501 .addUse(ResTypeReg) 2502 .addUse(IsLessEqReg) 2503 .addUse(NegOneOrZeroReg) // -1 or 0 2504 .addUse(buildOnesVal(false, ResType, I)) 2505 .constrainAllUses(TII, TRI, RBI); 2506 } 2507 2508 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2509 Register ResVReg, 2510 MachineInstr &I, 2511 const SPIRVType *IntTy, 2512 const SPIRVType *BoolTy) const { 2513 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2514 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2515 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2516 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2517 Register Zero = buildZerosVal(IntTy, I); 2518 Register One = buildOnesVal(false, IntTy, I); 2519 MachineBasicBlock &BB = *I.getParent(); 2520 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2521 .addDef(BitIntReg) 2522 .addUse(GR.getSPIRVTypeID(IntTy)) 2523 .addUse(IntReg) 2524 .addUse(One) 2525 .constrainAllUses(TII, TRI, RBI); 2526 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2527 .addDef(ResVReg) 2528 .addUse(GR.getSPIRVTypeID(BoolTy)) 2529 .addUse(BitIntReg) 2530 .addUse(Zero) 2531 .constrainAllUses(TII, TRI, RBI); 2532 } 2533 2534 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2535 const SPIRVType *ResType, 2536 MachineInstr &I) const { 2537 Register IntReg = I.getOperand(1).getReg(); 2538 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2539 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2540 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2541 if (ArgType == ResType) 2542 return BuildCOPY(ResVReg, IntReg, I); 2543 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2544 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2545 return selectUnOp(ResVReg, ResType, I, Opcode); 2546 } 2547 2548 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2549 const SPIRVType *ResType, 2550 const APInt &Imm, 2551 MachineInstr &I) const { 2552 unsigned TyOpcode = ResType->getOpcode(); 2553 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2554 MachineBasicBlock &BB = *I.getParent(); 2555 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2556 Imm.isZero()) 2557 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2558 .addDef(ResVReg) 2559 .addUse(GR.getSPIRVTypeID(ResType)) 2560 .constrainAllUses(TII, TRI, RBI); 2561 if (TyOpcode == SPIRV::OpTypeInt) { 2562 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2563 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2564 return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); 2565 } 2566 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2567 .addDef(ResVReg) 2568 .addUse(GR.getSPIRVTypeID(ResType)); 2569 // <=32-bit integers should be caught by the sdag pattern. 2570 assert(Imm.getBitWidth() > 32); 2571 addNumImm(Imm, MIB); 2572 return MIB.constrainAllUses(TII, TRI, RBI); 2573 } 2574 2575 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2576 const SPIRVType *ResType, 2577 MachineInstr &I) const { 2578 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2579 .addDef(ResVReg) 2580 .addUse(GR.getSPIRVTypeID(ResType)) 2581 .constrainAllUses(TII, TRI, RBI); 2582 } 2583 2584 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2585 assert(MO.isReg()); 2586 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2587 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2588 assert(TypeInst->getOperand(1).isReg()); 2589 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2590 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2591 } 2592 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2593 } 2594 2595 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2596 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2597 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2598 return TypeInst->getOperand(2).getImm(); 2599 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2600 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2601 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2602 } 2603 2604 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2605 const SPIRVType *ResType, 2606 MachineInstr &I) const { 2607 MachineBasicBlock &BB = *I.getParent(); 2608 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2609 .addDef(ResVReg) 2610 .addUse(GR.getSPIRVTypeID(ResType)) 2611 // object to insert 2612 .addUse(I.getOperand(3).getReg()) 2613 // composite to insert into 2614 .addUse(I.getOperand(2).getReg()); 2615 for (unsigned i = 4; i < I.getNumOperands(); i++) 2616 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2617 return MIB.constrainAllUses(TII, TRI, RBI); 2618 } 2619 2620 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2621 const SPIRVType *ResType, 2622 MachineInstr &I) const { 2623 MachineBasicBlock &BB = *I.getParent(); 2624 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2625 .addDef(ResVReg) 2626 .addUse(GR.getSPIRVTypeID(ResType)) 2627 .addUse(I.getOperand(2).getReg()); 2628 for (unsigned i = 3; i < I.getNumOperands(); i++) 2629 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2630 return MIB.constrainAllUses(TII, TRI, RBI); 2631 } 2632 2633 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2634 const SPIRVType *ResType, 2635 MachineInstr &I) const { 2636 if (isImm(I.getOperand(4), MRI)) 2637 return selectInsertVal(ResVReg, ResType, I); 2638 MachineBasicBlock &BB = *I.getParent(); 2639 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2640 .addDef(ResVReg) 2641 .addUse(GR.getSPIRVTypeID(ResType)) 2642 .addUse(I.getOperand(2).getReg()) 2643 .addUse(I.getOperand(3).getReg()) 2644 .addUse(I.getOperand(4).getReg()) 2645 .constrainAllUses(TII, TRI, RBI); 2646 } 2647 2648 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2649 const SPIRVType *ResType, 2650 MachineInstr &I) const { 2651 if (isImm(I.getOperand(3), MRI)) 2652 return selectExtractVal(ResVReg, ResType, I); 2653 MachineBasicBlock &BB = *I.getParent(); 2654 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2655 .addDef(ResVReg) 2656 .addUse(GR.getSPIRVTypeID(ResType)) 2657 .addUse(I.getOperand(2).getReg()) 2658 .addUse(I.getOperand(3).getReg()) 2659 .constrainAllUses(TII, TRI, RBI); 2660 } 2661 2662 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2663 const SPIRVType *ResType, 2664 MachineInstr &I) const { 2665 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2666 2667 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2668 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2669 // we have to use Op[InBounds]AccessChain. 2670 const unsigned Opcode = STI.isVulkanEnv() 2671 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2672 : SPIRV::OpAccessChain) 2673 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2674 : SPIRV::OpPtrAccessChain); 2675 2676 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2677 .addDef(ResVReg) 2678 .addUse(GR.getSPIRVTypeID(ResType)) 2679 // Object to get a pointer to. 2680 .addUse(I.getOperand(3).getReg()); 2681 // Adding indices. 2682 const unsigned StartingIndex = 2683 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2684 ? 5 2685 : 4; 2686 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2687 Res.addUse(I.getOperand(i).getReg()); 2688 return Res.constrainAllUses(TII, TRI, RBI); 2689 } 2690 2691 // Maybe wrap a value into OpSpecConstantOp 2692 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2693 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2694 bool Result = true; 2695 unsigned Lim = I.getNumExplicitOperands(); 2696 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2697 Register OpReg = I.getOperand(i).getReg(); 2698 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2699 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2700 SmallPtrSet<SPIRVType *, 4> Visited; 2701 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2702 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2703 GR.isAggregateType(OpType)) { 2704 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2705 // by selectAddrSpaceCast() 2706 CompositeArgs.push_back(OpReg); 2707 continue; 2708 } 2709 MachineFunction *MF = I.getMF(); 2710 Register WrapReg = GR.find(OpDefine, MF); 2711 if (WrapReg.isValid()) { 2712 CompositeArgs.push_back(WrapReg); 2713 continue; 2714 } 2715 // Create a new register for the wrapper 2716 WrapReg = MRI->createVirtualRegister(GR.getRegClass(OpType)); 2717 GR.add(OpDefine, MF, WrapReg); 2718 CompositeArgs.push_back(WrapReg); 2719 // Decorate the wrapper register and generate a new instruction 2720 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2721 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2722 MachineBasicBlock &BB = *I.getParent(); 2723 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2724 .addDef(WrapReg) 2725 .addUse(GR.getSPIRVTypeID(OpType)) 2726 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2727 .addUse(OpReg) 2728 .constrainAllUses(TII, TRI, RBI); 2729 if (!Result) 2730 break; 2731 } 2732 return Result; 2733 } 2734 2735 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2736 const SPIRVType *ResType, 2737 MachineInstr &I) const { 2738 MachineBasicBlock &BB = *I.getParent(); 2739 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2740 switch (IID) { 2741 case Intrinsic::spv_load: 2742 return selectLoad(ResVReg, ResType, I); 2743 case Intrinsic::spv_store: 2744 return selectStore(I); 2745 case Intrinsic::spv_extractv: 2746 return selectExtractVal(ResVReg, ResType, I); 2747 case Intrinsic::spv_insertv: 2748 return selectInsertVal(ResVReg, ResType, I); 2749 case Intrinsic::spv_extractelt: 2750 return selectExtractElt(ResVReg, ResType, I); 2751 case Intrinsic::spv_insertelt: 2752 return selectInsertElt(ResVReg, ResType, I); 2753 case Intrinsic::spv_gep: 2754 return selectGEP(ResVReg, ResType, I); 2755 case Intrinsic::spv_unref_global: 2756 case Intrinsic::spv_init_global: { 2757 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2758 MachineInstr *Init = I.getNumExplicitOperands() > 2 2759 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2760 : nullptr; 2761 assert(MI); 2762 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2763 } 2764 case Intrinsic::spv_undef: { 2765 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2766 .addDef(ResVReg) 2767 .addUse(GR.getSPIRVTypeID(ResType)); 2768 return MIB.constrainAllUses(TII, TRI, RBI); 2769 } 2770 case Intrinsic::spv_const_composite: { 2771 // If no values are attached, the composite is null constant. 2772 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2773 // Select a proper instruction. 2774 unsigned Opcode = SPIRV::OpConstantNull; 2775 SmallVector<Register> CompositeArgs; 2776 if (!IsNull) { 2777 Opcode = SPIRV::OpConstantComposite; 2778 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2779 return false; 2780 } 2781 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 2782 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2783 .addDef(ResVReg) 2784 .addUse(GR.getSPIRVTypeID(ResType)); 2785 // skip type MD node we already used when generated assign.type for this 2786 if (!IsNull) { 2787 for (Register OpReg : CompositeArgs) 2788 MIB.addUse(OpReg); 2789 } 2790 return MIB.constrainAllUses(TII, TRI, RBI); 2791 } 2792 case Intrinsic::spv_assign_name: { 2793 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2794 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2795 for (unsigned i = I.getNumExplicitDefs() + 2; 2796 i < I.getNumExplicitOperands(); ++i) { 2797 MIB.addImm(I.getOperand(i).getImm()); 2798 } 2799 return MIB.constrainAllUses(TII, TRI, RBI); 2800 } 2801 case Intrinsic::spv_switch: { 2802 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2803 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2804 if (I.getOperand(i).isReg()) 2805 MIB.addReg(I.getOperand(i).getReg()); 2806 else if (I.getOperand(i).isCImm()) 2807 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2808 else if (I.getOperand(i).isMBB()) 2809 MIB.addMBB(I.getOperand(i).getMBB()); 2810 else 2811 llvm_unreachable("Unexpected OpSwitch operand"); 2812 } 2813 return MIB.constrainAllUses(TII, TRI, RBI); 2814 } 2815 case Intrinsic::spv_loop_merge: 2816 case Intrinsic::spv_selection_merge: { 2817 const auto Opcode = IID == Intrinsic::spv_selection_merge 2818 ? SPIRV::OpSelectionMerge 2819 : SPIRV::OpLoopMerge; 2820 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2821 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2822 assert(I.getOperand(i).isMBB()); 2823 MIB.addMBB(I.getOperand(i).getMBB()); 2824 } 2825 MIB.addImm(SPIRV::SelectionControl::None); 2826 return MIB.constrainAllUses(TII, TRI, RBI); 2827 } 2828 case Intrinsic::spv_cmpxchg: 2829 return selectAtomicCmpXchg(ResVReg, ResType, I); 2830 case Intrinsic::spv_unreachable: 2831 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2832 .constrainAllUses(TII, TRI, RBI); 2833 case Intrinsic::spv_alloca: 2834 return selectFrameIndex(ResVReg, ResType, I); 2835 case Intrinsic::spv_alloca_array: 2836 return selectAllocaArray(ResVReg, ResType, I); 2837 case Intrinsic::spv_assume: 2838 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2839 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2840 .addUse(I.getOperand(1).getReg()) 2841 .constrainAllUses(TII, TRI, RBI); 2842 break; 2843 case Intrinsic::spv_expect: 2844 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2845 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2846 .addDef(ResVReg) 2847 .addUse(GR.getSPIRVTypeID(ResType)) 2848 .addUse(I.getOperand(2).getReg()) 2849 .addUse(I.getOperand(3).getReg()) 2850 .constrainAllUses(TII, TRI, RBI); 2851 break; 2852 case Intrinsic::arithmetic_fence: 2853 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2854 return BuildMI(BB, I, I.getDebugLoc(), 2855 TII.get(SPIRV::OpArithmeticFenceEXT)) 2856 .addDef(ResVReg) 2857 .addUse(GR.getSPIRVTypeID(ResType)) 2858 .addUse(I.getOperand(2).getReg()) 2859 .constrainAllUses(TII, TRI, RBI); 2860 else 2861 return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); 2862 break; 2863 case Intrinsic::spv_thread_id: 2864 // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id 2865 // intrinsic in LLVM IR for SPIR-V backend. 2866 // 2867 // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a 2868 // `GlobalInvocationId` builtin variable 2869 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, 2870 ResType, I); 2871 case Intrinsic::spv_thread_id_in_group: 2872 // The HLSL SV_GroupThreadId semantic is lowered to 2873 // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. 2874 // 2875 // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly 2876 // translated to a `LocalInvocationId` builtin variable 2877 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, 2878 ResType, I); 2879 case Intrinsic::spv_fdot: 2880 return selectFloatDot(ResVReg, ResType, I); 2881 case Intrinsic::spv_udot: 2882 case Intrinsic::spv_sdot: 2883 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2884 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2885 return selectIntegerDot(ResVReg, ResType, I, 2886 /*Signed=*/IID == Intrinsic::spv_sdot); 2887 return selectIntegerDotExpansion(ResVReg, ResType, I); 2888 case Intrinsic::spv_dot4add_i8packed: 2889 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2890 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2891 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2892 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2893 case Intrinsic::spv_dot4add_u8packed: 2894 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2895 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2896 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2897 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2898 case Intrinsic::spv_all: 2899 return selectAll(ResVReg, ResType, I); 2900 case Intrinsic::spv_any: 2901 return selectAny(ResVReg, ResType, I); 2902 case Intrinsic::spv_cross: 2903 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2904 case Intrinsic::spv_lerp: 2905 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2906 case Intrinsic::spv_length: 2907 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2908 case Intrinsic::spv_degrees: 2909 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2910 case Intrinsic::spv_frac: 2911 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2912 case Intrinsic::spv_normalize: 2913 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2914 case Intrinsic::spv_rsqrt: 2915 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2916 case Intrinsic::spv_sign: 2917 return selectSign(ResVReg, ResType, I); 2918 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2919 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2920 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2921 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2922 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2923 bool Result = true; 2924 auto MemSemConstant = 2925 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2926 Register MemSemReg = MemSemConstant.first; 2927 Result &= MemSemConstant.second; 2928 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2929 Register ScopeReg = ScopeConstant.first; 2930 Result &= ScopeConstant.second; 2931 MachineBasicBlock &BB = *I.getParent(); 2932 return Result && 2933 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2934 .addUse(ScopeReg) 2935 .addUse(ScopeReg) 2936 .addUse(MemSemReg) 2937 .constrainAllUses(TII, TRI, RBI); 2938 } 2939 case Intrinsic::spv_lifetime_start: 2940 case Intrinsic::spv_lifetime_end: { 2941 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2942 : SPIRV::OpLifetimeStop; 2943 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2944 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2945 if (Size == -1) 2946 Size = 0; 2947 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2948 .addUse(PtrReg) 2949 .addImm(Size) 2950 .constrainAllUses(TII, TRI, RBI); 2951 } 2952 case Intrinsic::spv_saturate: 2953 return selectSaturate(ResVReg, ResType, I); 2954 case Intrinsic::spv_nclamp: 2955 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2956 case Intrinsic::spv_uclamp: 2957 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2958 case Intrinsic::spv_sclamp: 2959 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2960 case Intrinsic::spv_wave_active_countbits: 2961 return selectWaveActiveCountBits(ResVReg, ResType, I); 2962 case Intrinsic::spv_wave_all: 2963 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); 2964 case Intrinsic::spv_wave_any: 2965 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); 2966 case Intrinsic::spv_wave_is_first_lane: 2967 return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); 2968 case Intrinsic::spv_wave_readlane: 2969 return selectWaveOpInst(ResVReg, ResType, I, 2970 SPIRV::OpGroupNonUniformShuffle); 2971 case Intrinsic::spv_step: 2972 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2973 case Intrinsic::spv_radians: 2974 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2975 // Discard intrinsics which we do not expect to actually represent code after 2976 // lowering or intrinsics which are not implemented but should not crash when 2977 // found in a customer's LLVM IR input. 2978 case Intrinsic::instrprof_increment: 2979 case Intrinsic::instrprof_increment_step: 2980 case Intrinsic::instrprof_value_profile: 2981 break; 2982 // Discard internal intrinsics. 2983 case Intrinsic::spv_value_md: 2984 break; 2985 case Intrinsic::spv_handle_fromBinding: { 2986 return selectHandleFromBinding(ResVReg, ResType, I); 2987 } 2988 case Intrinsic::spv_typedBufferStore: { 2989 selectImageWriteIntrinsic(I); 2990 return true; 2991 } 2992 case Intrinsic::spv_typedBufferLoad: { 2993 selectReadImageIntrinsic(ResVReg, ResType, I); 2994 return true; 2995 } 2996 case Intrinsic::spv_discard: { 2997 return selectDiscard(ResVReg, ResType, I); 2998 } 2999 default: { 3000 std::string DiagMsg; 3001 raw_string_ostream OS(DiagMsg); 3002 I.print(OS); 3003 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 3004 report_fatal_error(DiagMsg.c_str(), false); 3005 } 3006 } 3007 return true; 3008 } 3009 3010 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 3011 const SPIRVType *ResType, 3012 MachineInstr &I) const { 3013 3014 uint32_t Set = foldImm(I.getOperand(2), MRI); 3015 uint32_t Binding = foldImm(I.getOperand(3), MRI); 3016 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 3017 Register IndexReg = I.getOperand(5).getReg(); 3018 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 3019 3020 MachineIRBuilder MIRBuilder(I); 3021 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 3022 IndexReg, IsNonUniform, MIRBuilder); 3023 3024 if (IsNonUniform) 3025 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 3026 3027 // TODO: For now we assume the resource is an image, which needs to be 3028 // loaded to get the handle. That will not be true for storage buffers. 3029 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3030 .addDef(ResVReg) 3031 .addUse(GR.getSPIRVTypeID(ResType)) 3032 .addUse(VarReg) 3033 .constrainAllUses(TII, TRI, RBI); 3034 } 3035 3036 void SPIRVInstructionSelector::selectReadImageIntrinsic( 3037 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 3038 3039 // If the load of the image is in a different basic block, then 3040 // this will generate invalid code. A proper solution is to move 3041 // the OpLoad from selectHandleFromBinding here. However, to do 3042 // that we will need to change the return type of the intrinsic. 3043 // We will do that when we can, but for now trying to move forward with other 3044 // issues. 3045 Register ImageReg = I.getOperand(2).getReg(); 3046 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3047 "The image must be loaded in the same basic block as its use."); 3048 3049 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3050 if (ResultSize == 4) { 3051 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3052 .addDef(ResVReg) 3053 .addUse(GR.getSPIRVTypeID(ResType)) 3054 .addUse(ImageReg) 3055 .addUse(I.getOperand(3).getReg()); 3056 return; 3057 } 3058 3059 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 3060 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 3061 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 3062 .addDef(ReadReg) 3063 .addUse(GR.getSPIRVTypeID(ReadType)) 3064 .addUse(ImageReg) 3065 .addUse(I.getOperand(3).getReg()); 3066 3067 if (ResultSize == 1) { 3068 BuildMI(*I.getParent(), I, I.getDebugLoc(), 3069 TII.get(SPIRV::OpCompositeExtract)) 3070 .addDef(ResVReg) 3071 .addUse(GR.getSPIRVTypeID(ResType)) 3072 .addUse(ReadReg) 3073 .addImm(0); 3074 return; 3075 } 3076 extractSubvector(ResVReg, ResType, ReadReg, I); 3077 } 3078 3079 void SPIRVInstructionSelector::extractSubvector( 3080 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 3081 MachineInstr &InsertionPoint) const { 3082 SPIRVType *InputType = GR.getResultType(ReadReg); 3083 [[maybe_unused]] uint64_t InputSize = 3084 GR.getScalarOrVectorComponentCount(InputType); 3085 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 3086 assert(InputSize > 1 && "The input must be a vector."); 3087 assert(ResultSize > 1 && "The result must be a vector."); 3088 assert(ResultSize < InputSize && 3089 "Cannot extract more element than there are in the input."); 3090 SmallVector<Register> ComponentRegisters; 3091 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 3092 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 3093 for (uint64_t I = 0; I < ResultSize; I++) { 3094 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 3095 BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3096 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3097 .addDef(ComponentReg) 3098 .addUse(ScalarType->getOperand(0).getReg()) 3099 .addUse(ReadReg) 3100 .addImm(I); 3101 ComponentRegisters.emplace_back(ComponentReg); 3102 } 3103 3104 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 3105 InsertionPoint.getDebugLoc(), 3106 TII.get(SPIRV::OpCompositeConstruct)) 3107 .addDef(ResVReg) 3108 .addUse(GR.getSPIRVTypeID(ResType)); 3109 3110 for (Register ComponentReg : ComponentRegisters) 3111 MIB.addUse(ComponentReg); 3112 } 3113 3114 void SPIRVInstructionSelector::selectImageWriteIntrinsic( 3115 MachineInstr &I) const { 3116 // If the load of the image is in a different basic block, then 3117 // this will generate invalid code. A proper solution is to move 3118 // the OpLoad from selectHandleFromBinding here. However, to do 3119 // that we will need to change the return type of the intrinsic. 3120 // We will do that when we can, but for now trying to move forward with other 3121 // issues. 3122 Register ImageReg = I.getOperand(1).getReg(); 3123 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 3124 "The image must be loaded in the same basic block as its use."); 3125 Register CoordinateReg = I.getOperand(2).getReg(); 3126 Register DataReg = I.getOperand(3).getReg(); 3127 assert(GR.getResultType(DataReg)->getOpcode() == SPIRV::OpTypeVector); 3128 assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); 3129 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite)) 3130 .addUse(ImageReg) 3131 .addUse(CoordinateReg) 3132 .addUse(DataReg); 3133 } 3134 3135 Register SPIRVInstructionSelector::buildPointerToResource( 3136 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 3137 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 3138 MachineIRBuilder MIRBuilder) const { 3139 if (ArraySize == 1) 3140 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 3141 MIRBuilder); 3142 3143 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 3144 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 3145 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 3146 VarType, Set, Binding, MIRBuilder); 3147 3148 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 3149 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 3150 3151 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3152 if (IsNonUniform) { 3153 // It is unclear which value needs to be marked an non-uniform, so both 3154 // the index and the access changed are decorated as non-uniform. 3155 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3156 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 3157 } 3158 3159 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 3160 .addDef(AcReg) 3161 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3162 .addUse(VarReg) 3163 .addUse(IndexReg); 3164 3165 return AcReg; 3166 } 3167 3168 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3169 const SPIRVType *ResType, 3170 MachineInstr &I, 3171 bool IsSigned) const { 3172 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3173 // zero or sign extend 3174 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3175 bool Result = 3176 selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, Opcode); 3177 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3178 } 3179 3180 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3181 const SPIRVType *ResType, 3182 MachineInstr &I, 3183 Register SrcReg, 3184 bool IsSigned) const { 3185 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3186 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3187 .addDef(ResVReg) 3188 .addUse(GR.getSPIRVTypeID(ResType)) 3189 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3190 .addImm(Opcode) 3191 .addUse(SrcReg) 3192 .constrainAllUses(TII, TRI, RBI); 3193 } 3194 3195 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3196 const SPIRVType *ResType, 3197 MachineInstr &I, 3198 bool IsSigned) const { 3199 Register OpReg = I.getOperand(2).getReg(); 3200 // 1. split our int64 into 2 pieces using a bitcast 3201 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3202 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3203 MachineIRBuilder MIRBuilder(I); 3204 SPIRVType *postCastT = 3205 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3206 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3207 bool Result = 3208 selectOpWithSrcs(bitcastReg, postCastT, I, {OpReg}, SPIRV::OpBitcast); 3209 3210 // 2. call firstbithigh 3211 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3212 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3213 3214 // 3. split result vector into high bits and low bits 3215 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3216 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3217 3218 bool ZeroAsNull = STI.isOpenCLEnv(); 3219 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3220 if (isScalarRes) { 3221 // if scalar do a vector extract 3222 Result &= selectOpWithSrcs( 3223 HighReg, ResType, I, 3224 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3225 SPIRV::OpVectorExtractDynamic); 3226 Result &= selectOpWithSrcs( 3227 LowReg, ResType, I, 3228 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3229 SPIRV::OpVectorExtractDynamic); 3230 } else { // vector case do a shufflevector 3231 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3232 TII.get(SPIRV::OpVectorShuffle)) 3233 .addDef(HighReg) 3234 .addUse(GR.getSPIRVTypeID(ResType)) 3235 .addUse(FBHReg) 3236 .addUse(FBHReg); 3237 // ^^ this vector will not be selected from; could be empty 3238 unsigned j; 3239 for (j = 0; j < count * 2; j += 2) { 3240 MIB.addImm(j); 3241 } 3242 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3243 3244 // get low bits 3245 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3246 TII.get(SPIRV::OpVectorShuffle)) 3247 .addDef(LowReg) 3248 .addUse(GR.getSPIRVTypeID(ResType)) 3249 .addUse(FBHReg) 3250 .addUse(FBHReg); 3251 // ^^ this vector will not be selected from; could be empty 3252 for (j = 1; j < count * 2; j += 2) { 3253 MIB.addImm(j); 3254 } 3255 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3256 } 3257 3258 // 4. check if result of each top 32 bits is == -1 3259 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3260 Register NegOneReg; 3261 Register Reg0; 3262 Register Reg32; 3263 unsigned selectOp; 3264 unsigned addOp; 3265 if (isScalarRes) { 3266 NegOneReg = 3267 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3268 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3269 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3270 selectOp = SPIRV::OpSelectSISCond; 3271 addOp = SPIRV::OpIAddS; 3272 } else { 3273 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3274 NegOneReg = 3275 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3276 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3277 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3278 selectOp = SPIRV::OpSelectVIVCond; 3279 addOp = SPIRV::OpIAddV; 3280 } 3281 3282 // check if the high bits are == -1; true if -1 3283 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3284 Result &= selectOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3285 SPIRV::OpIEqual); 3286 3287 // Select low bits if true in BReg, otherwise high bits 3288 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3289 Result &= 3290 selectOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, selectOp); 3291 3292 // Add 32 for high bits, 0 for low bits 3293 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3294 Result &= selectOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3295 3296 return Result && 3297 selectOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3298 } 3299 3300 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3301 const SPIRVType *ResType, 3302 MachineInstr &I, 3303 bool IsSigned) const { 3304 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3305 Register OpReg = I.getOperand(2).getReg(); 3306 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3307 3308 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3309 case 16: 3310 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3311 case 32: 3312 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3313 case 64: 3314 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3315 default: 3316 report_fatal_error( 3317 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3318 } 3319 } 3320 3321 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3322 const SPIRVType *ResType, 3323 MachineInstr &I) const { 3324 // there was an allocation size parameter to the allocation instruction 3325 // that is not 1 3326 MachineBasicBlock &BB = *I.getParent(); 3327 bool Res = BuildMI(BB, I, I.getDebugLoc(), 3328 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3329 .addDef(ResVReg) 3330 .addUse(GR.getSPIRVTypeID(ResType)) 3331 .addUse(I.getOperand(2).getReg()) 3332 .constrainAllUses(TII, TRI, RBI); 3333 if (!STI.isVulkanEnv()) { 3334 unsigned Alignment = I.getOperand(3).getImm(); 3335 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); 3336 } 3337 return Res; 3338 } 3339 3340 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3341 const SPIRVType *ResType, 3342 MachineInstr &I) const { 3343 // Change order of instructions if needed: all OpVariable instructions in a 3344 // function must be the first instructions in the first block 3345 auto It = getOpVariableMBBIt(I); 3346 bool Res = BuildMI(*It->getParent(), It, It->getDebugLoc(), 3347 TII.get(SPIRV::OpVariable)) 3348 .addDef(ResVReg) 3349 .addUse(GR.getSPIRVTypeID(ResType)) 3350 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3351 .constrainAllUses(TII, TRI, RBI); 3352 if (!STI.isVulkanEnv()) { 3353 unsigned Alignment = I.getOperand(2).getImm(); 3354 buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, 3355 {Alignment}); 3356 } 3357 return Res; 3358 } 3359 3360 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3361 // InstructionSelector walks backwards through the instructions. We can use 3362 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3363 // first, so can generate an OpBranchConditional here. If there is no 3364 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3365 const MachineInstr *PrevI = I.getPrevNode(); 3366 MachineBasicBlock &MBB = *I.getParent(); 3367 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3368 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3369 .addUse(PrevI->getOperand(0).getReg()) 3370 .addMBB(PrevI->getOperand(1).getMBB()) 3371 .addMBB(I.getOperand(0).getMBB()) 3372 .constrainAllUses(TII, TRI, RBI); 3373 } 3374 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3375 .addMBB(I.getOperand(0).getMBB()) 3376 .constrainAllUses(TII, TRI, RBI); 3377 } 3378 3379 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3380 // InstructionSelector walks backwards through the instructions. For an 3381 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3382 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3383 // generate the OpBranchConditional in selectBranch above. 3384 // 3385 // If an OpBranchConditional has been generated, we simply return, as the work 3386 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3387 // implicit fallthrough to the next basic block, so we need to create an 3388 // OpBranchConditional with an explicit "false" argument pointing to the next 3389 // basic block that LLVM would fall through to. 3390 const MachineInstr *NextI = I.getNextNode(); 3391 // Check if this has already been successfully selected. 3392 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3393 return true; 3394 // Must be relying on implicit block fallthrough, so generate an 3395 // OpBranchConditional with the "next" basic block as the "false" target. 3396 MachineBasicBlock &MBB = *I.getParent(); 3397 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3398 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3399 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3400 .addUse(I.getOperand(0).getReg()) 3401 .addMBB(I.getOperand(1).getMBB()) 3402 .addMBB(NextMBB) 3403 .constrainAllUses(TII, TRI, RBI); 3404 } 3405 3406 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3407 const SPIRVType *ResType, 3408 MachineInstr &I) const { 3409 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3410 .addDef(ResVReg) 3411 .addUse(GR.getSPIRVTypeID(ResType)); 3412 const unsigned NumOps = I.getNumOperands(); 3413 for (unsigned i = 1; i < NumOps; i += 2) { 3414 MIB.addUse(I.getOperand(i + 0).getReg()); 3415 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3416 } 3417 bool Res = MIB.constrainAllUses(TII, TRI, RBI); 3418 MIB->setDesc(TII.get(TargetOpcode::PHI)); 3419 MIB->removeOperand(1); 3420 return Res; 3421 } 3422 3423 bool SPIRVInstructionSelector::selectGlobalValue( 3424 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3425 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3426 MachineIRBuilder MIRBuilder(I); 3427 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3428 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3429 SPIRVType *PointerBaseType; 3430 if (GVType->isArrayTy()) { 3431 SPIRVType *ArrayElementType = 3432 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3433 SPIRV::AccessQualifier::ReadWrite, false); 3434 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3435 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3436 } else { 3437 PointerBaseType = GR.getOrCreateSPIRVType( 3438 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3439 } 3440 3441 std::string GlobalIdent; 3442 if (!GV->hasName()) { 3443 unsigned &ID = UnnamedGlobalIDs[GV]; 3444 if (ID == 0) 3445 ID = UnnamedGlobalIDs.size(); 3446 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3447 } else { 3448 GlobalIdent = GV->getGlobalIdentifier(); 3449 } 3450 3451 // Behaviour of functions as operands depends on availability of the 3452 // corresponding extension (SPV_INTEL_function_pointers): 3453 // - If there is an extension to operate with functions as operands: 3454 // We create a proper constant operand and evaluate a correct type for a 3455 // function pointer. 3456 // - Without the required extension: 3457 // We have functions as operands in tests with blocks of instruction e.g. in 3458 // transcoding/global_block.ll. These operands are not used and should be 3459 // substituted by zero constants. Their type is expected to be always 3460 // OpTypePointer Function %uchar. 3461 if (isa<Function>(GV)) { 3462 const Constant *ConstVal = GV; 3463 MachineBasicBlock &BB = *I.getParent(); 3464 Register NewReg = GR.find(ConstVal, GR.CurMF); 3465 if (!NewReg.isValid()) { 3466 Register NewReg = ResVReg; 3467 GR.add(ConstVal, GR.CurMF, NewReg); 3468 const Function *GVFun = 3469 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3470 ? dyn_cast<Function>(GV) 3471 : nullptr; 3472 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3473 PointerBaseType, I, TII, 3474 GVFun ? SPIRV::StorageClass::CodeSectionINTEL 3475 : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3476 if (GVFun) { 3477 // References to a function via function pointers generate virtual 3478 // registers without a definition. We will resolve it later, during 3479 // module analysis stage. 3480 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3481 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3482 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3483 MachineInstrBuilder MB = 3484 BuildMI(BB, I, I.getDebugLoc(), 3485 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3486 .addDef(NewReg) 3487 .addUse(GR.getSPIRVTypeID(ResType)) 3488 .addUse(FuncVReg); 3489 // mapping the function pointer to the used Function 3490 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3491 return MB.constrainAllUses(TII, TRI, RBI); 3492 } 3493 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3494 .addDef(NewReg) 3495 .addUse(GR.getSPIRVTypeID(ResType)) 3496 .constrainAllUses(TII, TRI, RBI); 3497 } 3498 assert(NewReg != ResVReg); 3499 return BuildCOPY(ResVReg, NewReg, I); 3500 } 3501 auto GlobalVar = cast<GlobalVariable>(GV); 3502 assert(GlobalVar->getName() != "llvm.global.annotations"); 3503 3504 bool HasInit = GlobalVar->hasInitializer() && 3505 !isa<UndefValue>(GlobalVar->getInitializer()); 3506 // Skip empty declaration for GVs with initilaizers till we get the decl with 3507 // passed initializer. 3508 if (HasInit && !Init) 3509 return true; 3510 3511 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage; 3512 SPIRV::LinkageType::LinkageType LnkType = 3513 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3514 ? SPIRV::LinkageType::Import 3515 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3516 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3517 ? SPIRV::LinkageType::LinkOnceODR 3518 : SPIRV::LinkageType::Export); 3519 3520 const unsigned AddrSpace = GV->getAddressSpace(); 3521 SPIRV::StorageClass::StorageClass StorageClass = 3522 addressSpaceToStorageClass(AddrSpace, STI); 3523 SPIRVType *ResType = 3524 GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass); 3525 Register Reg = GR.buildGlobalVariable( 3526 ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, 3527 GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); 3528 return Reg.isValid(); 3529 } 3530 3531 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3532 const SPIRVType *ResType, 3533 MachineInstr &I) const { 3534 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3535 return selectExtInst(ResVReg, ResType, I, CL::log10); 3536 } 3537 3538 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3539 // is implemented as: 3540 // log10(x) = log2(x) * (1 / log2(10)) 3541 // = log2(x) * 0.30103 3542 3543 MachineIRBuilder MIRBuilder(I); 3544 MachineBasicBlock &BB = *I.getParent(); 3545 3546 // Build log2(x). 3547 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3548 bool Result = 3549 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3550 .addDef(VarReg) 3551 .addUse(GR.getSPIRVTypeID(ResType)) 3552 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3553 .addImm(GL::Log2) 3554 .add(I.getOperand(1)) 3555 .constrainAllUses(TII, TRI, RBI); 3556 3557 // Build 0.30103. 3558 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3559 ResType->getOpcode() == SPIRV::OpTypeFloat); 3560 // TODO: Add matrix implementation once supported by the HLSL frontend. 3561 const SPIRVType *SpirvScalarType = 3562 ResType->getOpcode() == SPIRV::OpTypeVector 3563 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3564 : ResType; 3565 Register ScaleReg = 3566 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3567 3568 // Multiply log2(x) by 0.30103 to get log10(x) result. 3569 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3570 ? SPIRV::OpVectorTimesScalar 3571 : SPIRV::OpFMulS; 3572 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3573 .addDef(ResVReg) 3574 .addUse(GR.getSPIRVTypeID(ResType)) 3575 .addUse(VarReg) 3576 .addUse(ScaleReg) 3577 .constrainAllUses(TII, TRI, RBI); 3578 } 3579 3580 // Generate the instructions to load 3-element vector builtin input 3581 // IDs/Indices. 3582 // Like: GlobalInvocationId, LocalInvocationId, etc.... 3583 bool SPIRVInstructionSelector::loadVec3BuiltinInputID( 3584 SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, 3585 const SPIRVType *ResType, MachineInstr &I) const { 3586 MachineIRBuilder MIRBuilder(I); 3587 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3588 const SPIRVType *Vec3Ty = 3589 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3590 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3591 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3592 3593 // Create new register for the input ID builtin variable. 3594 Register NewRegister = 3595 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3596 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3597 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3598 3599 // Build global variable with the necessary decorations for the input ID 3600 // builtin variable. 3601 Register Variable = GR.buildGlobalVariable( 3602 NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, 3603 SPIRV::StorageClass::Input, nullptr, true, true, 3604 SPIRV::LinkageType::Import, MIRBuilder, false); 3605 3606 // Create new register for loading value. 3607 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3608 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3609 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3610 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3611 3612 // Load v3uint value from the global variable. 3613 bool Result = 3614 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3615 .addDef(LoadedRegister) 3616 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3617 .addUse(Variable); 3618 3619 // Get the input ID index. Expecting operand is a constant immediate value, 3620 // wrapped in a type assignment. 3621 assert(I.getOperand(2).isReg()); 3622 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3623 3624 // Extract the input ID from the loaded vector value. 3625 MachineBasicBlock &BB = *I.getParent(); 3626 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3627 .addDef(ResVReg) 3628 .addUse(GR.getSPIRVTypeID(ResType)) 3629 .addUse(LoadedRegister) 3630 .addImm(ThreadId); 3631 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3632 } 3633 3634 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3635 MachineInstr &I) const { 3636 MachineIRBuilder MIRBuilder(I); 3637 if (Type->getOpcode() != SPIRV::OpTypeVector) 3638 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3639 3640 uint64_t VectorSize = Type->getOperand(2).getImm(); 3641 if (VectorSize == 4) 3642 return Type; 3643 3644 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3645 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3646 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3647 } 3648 3649 namespace llvm { 3650 InstructionSelector * 3651 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3652 const SPIRVSubtarget &Subtarget, 3653 const RegisterBankInfo &RBI) { 3654 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3655 } 3656 } // namespace llvm 3657