1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/Register.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/IR/IntrinsicsSPIRV.h" 34 #include "llvm/Support/Debug.h" 35 36 #define DEBUG_TYPE "spirv-isel" 37 38 using namespace llvm; 39 namespace CL = SPIRV::OpenCLExtInst; 40 namespace GL = SPIRV::GLSLExtInst; 41 42 using ExtInstList = 43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 44 45 namespace { 46 47 #define GET_GLOBALISEL_PREDICATE_BITSET 48 #include "SPIRVGenGlobalISel.inc" 49 #undef GET_GLOBALISEL_PREDICATE_BITSET 50 51 class SPIRVInstructionSelector : public InstructionSelector { 52 const SPIRVSubtarget &STI; 53 const SPIRVInstrInfo &TII; 54 const SPIRVRegisterInfo &TRI; 55 const RegisterBankInfo &RBI; 56 SPIRVGlobalRegistry &GR; 57 MachineRegisterInfo *MRI; 58 MachineFunction *HasVRegsReset = nullptr; 59 60 /// We need to keep track of the number we give to anonymous global values to 61 /// generate the same name every time when this is needed. 62 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 63 64 public: 65 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 66 const SPIRVSubtarget &ST, 67 const RegisterBankInfo &RBI); 68 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 69 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 70 BlockFrequencyInfo *BFI) override; 71 // Common selection code. Instruction-specific selection occurs in spvSelect. 72 bool select(MachineInstr &I) override; 73 static const char *getName() { return DEBUG_TYPE; } 74 75 #define GET_GLOBALISEL_PREDICATES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_PREDICATES_DECL 78 79 #define GET_GLOBALISEL_TEMPORARIES_DECL 80 #include "SPIRVGenGlobalISel.inc" 81 #undef GET_GLOBALISEL_TEMPORARIES_DECL 82 83 private: 84 void resetVRegsType(MachineFunction &MF); 85 86 // tblgen-erated 'select' implementation, used as the initial selector for 87 // the patterns that don't require complex C++. 88 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 89 90 // All instruction-specific selection that didn't happen in "select()". 91 // Is basically a large Switch/Case delegating to all other select method. 92 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I) const; 94 95 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 96 MachineInstr &I, bool IsSigned) const; 97 98 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I, bool IsSigned) const; 100 101 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 102 MachineInstr &I, Register SrcReg, 103 bool IsSigned) const; 104 105 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 106 MachineInstr &I, bool IsSigned) const; 107 108 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 109 const MachineInstr *Init = nullptr) const; 110 111 bool selectNAryOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I, std::vector<Register> SrcRegs, 113 unsigned Opcode) const; 114 115 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, Register SrcReg, 117 unsigned Opcode) const; 118 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 119 unsigned Opcode) const; 120 121 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 125 MachineInstr &I) const; 126 bool selectStore(MachineInstr &I) const; 127 128 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 129 MachineInstr &I) const; 130 bool selectStackRestore(MachineInstr &I) const; 131 132 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 133 134 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 135 MachineInstr &I, unsigned NewOpcode, 136 unsigned NegateOpcode = 0) const; 137 138 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 139 MachineInstr &I) const; 140 141 bool selectFence(MachineInstr &I) const; 142 143 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 144 MachineInstr &I) const; 145 146 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 147 MachineInstr &I, unsigned OpType) const; 148 149 bool selectAll(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 152 bool selectAny(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 158 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 163 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 164 unsigned comparisonOpcode, MachineInstr &I) const; 165 bool selectCross(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I) const; 169 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I) const; 171 172 bool selectSign(Register ResVReg, const SPIRVType *ResType, 173 MachineInstr &I) const; 174 175 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 176 MachineInstr &I) const; 177 178 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 179 MachineInstr &I, unsigned Opcode) const; 180 181 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 template <bool Signed> 185 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 template <bool Signed> 188 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 192 int OpIdx) const; 193 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 194 int OpIdx) const; 195 196 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 197 MachineInstr &I) const; 198 199 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 200 bool IsSigned) const; 201 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 202 bool IsSigned, unsigned Opcode) const; 203 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 204 bool IsSigned) const; 205 206 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 207 MachineInstr &I) const; 208 209 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 210 const SPIRVType *intTy, const SPIRVType *boolTy) const; 211 212 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 215 MachineInstr &I) const; 216 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I) const; 218 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I) const; 220 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I) const; 224 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 227 MachineInstr &I) const; 228 229 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 230 MachineInstr &I) const; 231 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 232 MachineInstr &I) const; 233 234 bool selectBranch(MachineInstr &I) const; 235 bool selectBranchCond(MachineInstr &I) const; 236 237 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 240 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 241 MachineInstr &I, GL::GLSLExtInst GLInst) const; 242 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 243 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 244 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 245 MachineInstr &I, CL::OpenCLExtInst CLInst, 246 GL::GLSLExtInst GLInst) const; 247 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 248 MachineInstr &I, const ExtInstList &ExtInsts) const; 249 250 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 251 MachineInstr &I) const; 252 253 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 254 MachineInstr &I) const; 255 256 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 257 MachineInstr &I) const; 258 259 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 262 bool selectWaveReadLaneAt(Register ResVReg, const SPIRVType *ResType, 263 MachineInstr &I) const; 264 265 bool selectUnmergeValues(MachineInstr &I) const; 266 267 void selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 268 MachineInstr &I) const; 269 270 // Utilities 271 Register buildI32Constant(uint32_t Val, MachineInstr &I, 272 const SPIRVType *ResType = nullptr) const; 273 274 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 275 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 276 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 277 MachineInstr &I) const; 278 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 279 280 bool wrapIntoSpecConstantOp(MachineInstr &I, 281 SmallVector<Register> &CompositeArgs) const; 282 283 Register getUcharPtrTypeReg(MachineInstr &I, 284 SPIRV::StorageClass::StorageClass SC) const; 285 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 286 Register Src, Register DestType, 287 uint32_t Opcode) const; 288 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 289 SPIRVType *SrcPtrTy) const; 290 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 291 uint32_t Binding, uint32_t ArraySize, 292 Register IndexReg, bool IsNonUniform, 293 MachineIRBuilder MIRBuilder) const; 294 }; 295 296 } // end anonymous namespace 297 298 #define GET_GLOBALISEL_IMPL 299 #include "SPIRVGenGlobalISel.inc" 300 #undef GET_GLOBALISEL_IMPL 301 302 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 303 const SPIRVSubtarget &ST, 304 const RegisterBankInfo &RBI) 305 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 306 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 307 #define GET_GLOBALISEL_PREDICATES_INIT 308 #include "SPIRVGenGlobalISel.inc" 309 #undef GET_GLOBALISEL_PREDICATES_INIT 310 #define GET_GLOBALISEL_TEMPORARIES_INIT 311 #include "SPIRVGenGlobalISel.inc" 312 #undef GET_GLOBALISEL_TEMPORARIES_INIT 313 { 314 } 315 316 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 317 CodeGenCoverage *CoverageInfo, 318 ProfileSummaryInfo *PSI, 319 BlockFrequencyInfo *BFI) { 320 MRI = &MF.getRegInfo(); 321 GR.setCurrentFunc(MF); 322 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 323 } 324 325 // Ensure that register classes correspond to pattern matching rules. 326 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 327 if (HasVRegsReset == &MF) 328 return; 329 HasVRegsReset = &MF; 330 331 MachineRegisterInfo &MRI = MF.getRegInfo(); 332 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 333 Register Reg = Register::index2VirtReg(I); 334 LLT RegType = MRI.getType(Reg); 335 if (RegType.isScalar()) 336 MRI.setType(Reg, LLT::scalar(64)); 337 else if (RegType.isPointer()) 338 MRI.setType(Reg, LLT::pointer(0, 64)); 339 else if (RegType.isVector()) 340 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 341 } 342 for (const auto &MBB : MF) { 343 for (const auto &MI : MBB) { 344 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 345 continue; 346 Register DstReg = MI.getOperand(0).getReg(); 347 LLT DstType = MRI.getType(DstReg); 348 Register SrcReg = MI.getOperand(1).getReg(); 349 LLT SrcType = MRI.getType(SrcReg); 350 if (DstType != SrcType) 351 MRI.setType(DstReg, MRI.getType(SrcReg)); 352 353 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 354 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 355 if (DstRC != SrcRC && SrcRC) 356 MRI.setRegClass(DstReg, SrcRC); 357 } 358 } 359 } 360 361 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 362 363 // Defined in SPIRVLegalizerInfo.cpp. 364 extern bool isTypeFoldingSupported(unsigned Opcode); 365 366 bool SPIRVInstructionSelector::select(MachineInstr &I) { 367 resetVRegsType(*I.getParent()->getParent()); 368 369 assert(I.getParent() && "Instruction should be in a basic block!"); 370 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 371 372 Register Opcode = I.getOpcode(); 373 // If it's not a GMIR instruction, we've selected it already. 374 if (!isPreISelGenericOpcode(Opcode)) { 375 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 376 Register DstReg = I.getOperand(0).getReg(); 377 Register SrcReg = I.getOperand(1).getReg(); 378 auto *Def = MRI->getVRegDef(SrcReg); 379 if (isTypeFoldingSupported(Def->getOpcode())) { 380 bool Res = selectImpl(I, *CoverageInfo); 381 LLVM_DEBUG({ 382 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 383 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 384 I.print(dbgs()); 385 } 386 }); 387 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 388 if (Res) 389 return Res; 390 } 391 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 392 MRI->replaceRegWith(SrcReg, DstReg); 393 I.removeFromParent(); 394 return true; 395 } else if (I.getNumDefs() == 1) { 396 // Make all vregs 64 bits (for SPIR-V IDs). 397 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 398 } 399 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 400 } 401 402 if (I.getNumOperands() != I.getNumExplicitOperands()) { 403 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 404 return false; 405 } 406 407 // Common code for getting return reg+type, and removing selected instr 408 // from parent occurs here. Instr-specific selection happens in spvSelect(). 409 bool HasDefs = I.getNumDefs() > 0; 410 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 411 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 412 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 413 if (spvSelect(ResVReg, ResType, I)) { 414 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 415 for (unsigned i = 0; i < I.getNumDefs(); ++i) 416 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 417 I.removeFromParent(); 418 return true; 419 } 420 return false; 421 } 422 423 static bool mayApplyGenericSelection(unsigned Opcode) { 424 switch (Opcode) { 425 case TargetOpcode::G_CONSTANT: 426 return false; 427 case TargetOpcode::G_SADDO: 428 case TargetOpcode::G_SSUBO: 429 return true; 430 } 431 return isTypeFoldingSupported(Opcode); 432 } 433 434 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 435 const SPIRVType *ResType, 436 MachineInstr &I) const { 437 const unsigned Opcode = I.getOpcode(); 438 if (mayApplyGenericSelection(Opcode)) 439 return selectImpl(I, *CoverageInfo); 440 switch (Opcode) { 441 case TargetOpcode::G_CONSTANT: 442 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 443 I); 444 case TargetOpcode::G_GLOBAL_VALUE: 445 return selectGlobalValue(ResVReg, I); 446 case TargetOpcode::G_IMPLICIT_DEF: 447 return selectOpUndef(ResVReg, ResType, I); 448 case TargetOpcode::G_FREEZE: 449 return selectFreeze(ResVReg, ResType, I); 450 451 case TargetOpcode::G_INTRINSIC: 452 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 453 case TargetOpcode::G_INTRINSIC_CONVERGENT: 454 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 455 return selectIntrinsic(ResVReg, ResType, I); 456 case TargetOpcode::G_BITREVERSE: 457 return selectBitreverse(ResVReg, ResType, I); 458 459 case TargetOpcode::G_BUILD_VECTOR: 460 return selectBuildVector(ResVReg, ResType, I); 461 case TargetOpcode::G_SPLAT_VECTOR: 462 return selectSplatVector(ResVReg, ResType, I); 463 464 case TargetOpcode::G_SHUFFLE_VECTOR: { 465 MachineBasicBlock &BB = *I.getParent(); 466 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 467 .addDef(ResVReg) 468 .addUse(GR.getSPIRVTypeID(ResType)) 469 .addUse(I.getOperand(1).getReg()) 470 .addUse(I.getOperand(2).getReg()); 471 for (auto V : I.getOperand(3).getShuffleMask()) 472 MIB.addImm(V); 473 return MIB.constrainAllUses(TII, TRI, RBI); 474 } 475 case TargetOpcode::G_MEMMOVE: 476 case TargetOpcode::G_MEMCPY: 477 case TargetOpcode::G_MEMSET: 478 return selectMemOperation(ResVReg, I); 479 480 case TargetOpcode::G_ICMP: 481 return selectICmp(ResVReg, ResType, I); 482 case TargetOpcode::G_FCMP: 483 return selectFCmp(ResVReg, ResType, I); 484 485 case TargetOpcode::G_FRAME_INDEX: 486 return selectFrameIndex(ResVReg, ResType, I); 487 488 case TargetOpcode::G_LOAD: 489 return selectLoad(ResVReg, ResType, I); 490 case TargetOpcode::G_STORE: 491 return selectStore(I); 492 493 case TargetOpcode::G_BR: 494 return selectBranch(I); 495 case TargetOpcode::G_BRCOND: 496 return selectBranchCond(I); 497 498 case TargetOpcode::G_PHI: 499 return selectPhi(ResVReg, ResType, I); 500 501 case TargetOpcode::G_FPTOSI: 502 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 503 case TargetOpcode::G_FPTOUI: 504 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 505 506 case TargetOpcode::G_SITOFP: 507 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 508 case TargetOpcode::G_UITOFP: 509 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 510 511 case TargetOpcode::G_CTPOP: 512 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 513 case TargetOpcode::G_SMIN: 514 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 515 case TargetOpcode::G_UMIN: 516 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 517 518 case TargetOpcode::G_SMAX: 519 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 520 case TargetOpcode::G_UMAX: 521 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 522 523 case TargetOpcode::G_FMA: 524 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 525 526 case TargetOpcode::G_FPOW: 527 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 528 case TargetOpcode::G_FPOWI: 529 return selectExtInst(ResVReg, ResType, I, CL::pown); 530 531 case TargetOpcode::G_FEXP: 532 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 533 case TargetOpcode::G_FEXP2: 534 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 535 536 case TargetOpcode::G_FLOG: 537 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 538 case TargetOpcode::G_FLOG2: 539 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 540 case TargetOpcode::G_FLOG10: 541 return selectLog10(ResVReg, ResType, I); 542 543 case TargetOpcode::G_FABS: 544 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 545 case TargetOpcode::G_ABS: 546 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 547 548 case TargetOpcode::G_FMINNUM: 549 case TargetOpcode::G_FMINIMUM: 550 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 551 case TargetOpcode::G_FMAXNUM: 552 case TargetOpcode::G_FMAXIMUM: 553 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 554 555 case TargetOpcode::G_FCOPYSIGN: 556 return selectExtInst(ResVReg, ResType, I, CL::copysign); 557 558 case TargetOpcode::G_FCEIL: 559 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 560 case TargetOpcode::G_FFLOOR: 561 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 562 563 case TargetOpcode::G_FCOS: 564 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 565 case TargetOpcode::G_FSIN: 566 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 567 case TargetOpcode::G_FTAN: 568 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 569 case TargetOpcode::G_FACOS: 570 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 571 case TargetOpcode::G_FASIN: 572 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 573 case TargetOpcode::G_FATAN: 574 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 575 case TargetOpcode::G_FATAN2: 576 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 577 case TargetOpcode::G_FCOSH: 578 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 579 case TargetOpcode::G_FSINH: 580 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 581 case TargetOpcode::G_FTANH: 582 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 583 584 case TargetOpcode::G_FSQRT: 585 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 586 587 case TargetOpcode::G_CTTZ: 588 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 589 return selectExtInst(ResVReg, ResType, I, CL::ctz); 590 case TargetOpcode::G_CTLZ: 591 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 592 return selectExtInst(ResVReg, ResType, I, CL::clz); 593 594 case TargetOpcode::G_INTRINSIC_ROUND: 595 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 596 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 597 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 598 case TargetOpcode::G_INTRINSIC_TRUNC: 599 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 600 case TargetOpcode::G_FRINT: 601 case TargetOpcode::G_FNEARBYINT: 602 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 603 604 case TargetOpcode::G_SMULH: 605 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 606 case TargetOpcode::G_UMULH: 607 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 608 609 case TargetOpcode::G_SADDSAT: 610 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 611 case TargetOpcode::G_UADDSAT: 612 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 613 case TargetOpcode::G_SSUBSAT: 614 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 615 case TargetOpcode::G_USUBSAT: 616 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 617 618 case TargetOpcode::G_UADDO: 619 return selectOverflowArith(ResVReg, ResType, I, 620 ResType->getOpcode() == SPIRV::OpTypeVector 621 ? SPIRV::OpIAddCarryV 622 : SPIRV::OpIAddCarryS); 623 case TargetOpcode::G_USUBO: 624 return selectOverflowArith(ResVReg, ResType, I, 625 ResType->getOpcode() == SPIRV::OpTypeVector 626 ? SPIRV::OpISubBorrowV 627 : SPIRV::OpISubBorrowS); 628 case TargetOpcode::G_UMULO: 629 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 630 case TargetOpcode::G_SMULO: 631 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 632 633 case TargetOpcode::G_SEXT: 634 return selectExt(ResVReg, ResType, I, true); 635 case TargetOpcode::G_ANYEXT: 636 case TargetOpcode::G_ZEXT: 637 return selectExt(ResVReg, ResType, I, false); 638 case TargetOpcode::G_TRUNC: 639 return selectTrunc(ResVReg, ResType, I); 640 case TargetOpcode::G_FPTRUNC: 641 case TargetOpcode::G_FPEXT: 642 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 643 644 case TargetOpcode::G_PTRTOINT: 645 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 646 case TargetOpcode::G_INTTOPTR: 647 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 648 case TargetOpcode::G_BITCAST: 649 return selectBitcast(ResVReg, ResType, I); 650 case TargetOpcode::G_ADDRSPACE_CAST: 651 return selectAddrSpaceCast(ResVReg, ResType, I); 652 case TargetOpcode::G_PTR_ADD: { 653 // Currently, we get G_PTR_ADD only applied to global variables. 654 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 655 Register GV = I.getOperand(1).getReg(); 656 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 657 (void)II; 658 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 659 (*II).getOpcode() == TargetOpcode::COPY || 660 (*II).getOpcode() == SPIRV::OpVariable) && 661 isImm(I.getOperand(2), MRI)); 662 // It may be the initialization of a global variable. 663 bool IsGVInit = false; 664 for (MachineRegisterInfo::use_instr_iterator 665 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 666 UseEnd = MRI->use_instr_end(); 667 UseIt != UseEnd; UseIt = std::next(UseIt)) { 668 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 669 (*UseIt).getOpcode() == SPIRV::OpVariable) { 670 IsGVInit = true; 671 break; 672 } 673 } 674 MachineBasicBlock &BB = *I.getParent(); 675 if (!IsGVInit) { 676 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 677 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 678 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 679 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 680 // Build a new virtual register that is associated with the required 681 // data type. 682 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 683 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 684 // Having a correctly typed base we are ready to build the actually 685 // required GEP. It may not be a constant though, because all Operands 686 // of OpSpecConstantOp is to originate from other const instructions, 687 // and only the AccessChain named opcodes accept a global OpVariable 688 // instruction. We can't use an AccessChain opcode because of the type 689 // mismatch between result and base types. 690 if (!GR.isBitcastCompatible(ResType, GVType)) 691 report_fatal_error( 692 "incompatible result and operand types in a bitcast"); 693 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 694 MachineInstrBuilder MIB = 695 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 696 .addDef(NewVReg) 697 .addUse(ResTypeReg) 698 .addUse(GV); 699 return MIB.constrainAllUses(TII, TRI, RBI) && 700 BuildMI(BB, I, I.getDebugLoc(), 701 TII.get(STI.isVulkanEnv() 702 ? SPIRV::OpInBoundsAccessChain 703 : SPIRV::OpInBoundsPtrAccessChain)) 704 .addDef(ResVReg) 705 .addUse(ResTypeReg) 706 .addUse(NewVReg) 707 .addUse(I.getOperand(2).getReg()) 708 .constrainAllUses(TII, TRI, RBI); 709 } else { 710 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 711 .addDef(ResVReg) 712 .addUse(GR.getSPIRVTypeID(ResType)) 713 .addImm( 714 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 715 .addUse(GV) 716 .addUse(I.getOperand(2).getReg()) 717 .constrainAllUses(TII, TRI, RBI); 718 } 719 } 720 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 721 // initialize a global variable with a constant expression (e.g., the test 722 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 723 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 724 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 725 .addDef(ResVReg) 726 .addUse(GR.getSPIRVTypeID(ResType)) 727 .addImm(static_cast<uint32_t>( 728 SPIRV::Opcode::InBoundsPtrAccessChain)) 729 .addUse(GV) 730 .addUse(Idx) 731 .addUse(I.getOperand(2).getReg()); 732 return MIB.constrainAllUses(TII, TRI, RBI); 733 } 734 735 case TargetOpcode::G_ATOMICRMW_OR: 736 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 737 case TargetOpcode::G_ATOMICRMW_ADD: 738 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 739 case TargetOpcode::G_ATOMICRMW_AND: 740 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 741 case TargetOpcode::G_ATOMICRMW_MAX: 742 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 743 case TargetOpcode::G_ATOMICRMW_MIN: 744 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 745 case TargetOpcode::G_ATOMICRMW_SUB: 746 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 747 case TargetOpcode::G_ATOMICRMW_XOR: 748 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 749 case TargetOpcode::G_ATOMICRMW_UMAX: 750 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 751 case TargetOpcode::G_ATOMICRMW_UMIN: 752 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 753 case TargetOpcode::G_ATOMICRMW_XCHG: 754 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 755 case TargetOpcode::G_ATOMIC_CMPXCHG: 756 return selectAtomicCmpXchg(ResVReg, ResType, I); 757 758 case TargetOpcode::G_ATOMICRMW_FADD: 759 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 760 case TargetOpcode::G_ATOMICRMW_FSUB: 761 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 762 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 763 SPIRV::OpFNegate); 764 case TargetOpcode::G_ATOMICRMW_FMIN: 765 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 766 case TargetOpcode::G_ATOMICRMW_FMAX: 767 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 768 769 case TargetOpcode::G_FENCE: 770 return selectFence(I); 771 772 case TargetOpcode::G_STACKSAVE: 773 return selectStackSave(ResVReg, ResType, I); 774 case TargetOpcode::G_STACKRESTORE: 775 return selectStackRestore(I); 776 777 case TargetOpcode::G_UNMERGE_VALUES: 778 return selectUnmergeValues(I); 779 780 // Discard gen opcodes for intrinsics which we do not expect to actually 781 // represent code after lowering or intrinsics which are not implemented but 782 // should not crash when found in a customer's LLVM IR input. 783 case TargetOpcode::G_TRAP: 784 case TargetOpcode::G_DEBUGTRAP: 785 case TargetOpcode::G_UBSANTRAP: 786 case TargetOpcode::DBG_LABEL: 787 return true; 788 789 default: 790 return false; 791 } 792 } 793 794 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 795 const SPIRVType *ResType, 796 MachineInstr &I, 797 GL::GLSLExtInst GLInst) const { 798 return selectExtInst(ResVReg, ResType, I, 799 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 800 } 801 802 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 803 const SPIRVType *ResType, 804 MachineInstr &I, 805 CL::OpenCLExtInst CLInst) const { 806 return selectExtInst(ResVReg, ResType, I, 807 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 808 } 809 810 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 811 const SPIRVType *ResType, 812 MachineInstr &I, 813 CL::OpenCLExtInst CLInst, 814 GL::GLSLExtInst GLInst) const { 815 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 816 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 817 return selectExtInst(ResVReg, ResType, I, ExtInsts); 818 } 819 820 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 821 const SPIRVType *ResType, 822 MachineInstr &I, 823 const ExtInstList &Insts) const { 824 825 for (const auto &Ex : Insts) { 826 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 827 uint32_t Opcode = Ex.second; 828 if (STI.canUseExtInstSet(Set)) { 829 MachineBasicBlock &BB = *I.getParent(); 830 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 831 .addDef(ResVReg) 832 .addUse(GR.getSPIRVTypeID(ResType)) 833 .addImm(static_cast<uint32_t>(Set)) 834 .addImm(Opcode); 835 const unsigned NumOps = I.getNumOperands(); 836 unsigned Index = 1; 837 if (Index < NumOps && 838 I.getOperand(Index).getType() == 839 MachineOperand::MachineOperandType::MO_IntrinsicID) 840 Index = 2; 841 for (; Index < NumOps; ++Index) 842 MIB.add(I.getOperand(Index)); 843 return MIB.constrainAllUses(TII, TRI, RBI); 844 } 845 } 846 return false; 847 } 848 849 bool SPIRVInstructionSelector::selectNAryOpWithSrcs(Register ResVReg, 850 const SPIRVType *ResType, 851 MachineInstr &I, 852 std::vector<Register> Srcs, 853 unsigned Opcode) const { 854 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 855 .addDef(ResVReg) 856 .addUse(GR.getSPIRVTypeID(ResType)); 857 for (Register SReg : Srcs) { 858 MIB.addUse(SReg); 859 } 860 return MIB.constrainAllUses(TII, TRI, RBI); 861 } 862 863 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 864 const SPIRVType *ResType, 865 MachineInstr &I, 866 Register SrcReg, 867 unsigned Opcode) const { 868 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 869 .addDef(ResVReg) 870 .addUse(GR.getSPIRVTypeID(ResType)) 871 .addUse(SrcReg) 872 .constrainAllUses(TII, TRI, RBI); 873 } 874 875 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 876 const SPIRVType *ResType, 877 MachineInstr &I, 878 unsigned Opcode) const { 879 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 880 Register SrcReg = I.getOperand(1).getReg(); 881 bool IsGV = false; 882 for (MachineRegisterInfo::def_instr_iterator DefIt = 883 MRI->def_instr_begin(SrcReg); 884 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 885 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 886 IsGV = true; 887 break; 888 } 889 } 890 if (IsGV) { 891 uint32_t SpecOpcode = 0; 892 switch (Opcode) { 893 case SPIRV::OpConvertPtrToU: 894 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 895 break; 896 case SPIRV::OpConvertUToPtr: 897 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 898 break; 899 } 900 if (SpecOpcode) 901 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 902 TII.get(SPIRV::OpSpecConstantOp)) 903 .addDef(ResVReg) 904 .addUse(GR.getSPIRVTypeID(ResType)) 905 .addImm(SpecOpcode) 906 .addUse(SrcReg) 907 .constrainAllUses(TII, TRI, RBI); 908 } 909 } 910 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 911 Opcode); 912 } 913 914 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 915 const SPIRVType *ResType, 916 MachineInstr &I) const { 917 Register OpReg = I.getOperand(1).getReg(); 918 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 919 if (!GR.isBitcastCompatible(ResType, OpType)) 920 report_fatal_error("incompatible result and operand types in a bitcast"); 921 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 922 } 923 924 static void addMemoryOperands(MachineMemOperand *MemOp, 925 MachineInstrBuilder &MIB) { 926 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 927 if (MemOp->isVolatile()) 928 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 929 if (MemOp->isNonTemporal()) 930 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 931 if (MemOp->getAlign().value()) 932 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 933 934 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 935 MIB.addImm(SpvMemOp); 936 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 937 MIB.addImm(MemOp->getAlign().value()); 938 } 939 } 940 941 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 942 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 943 if (Flags & MachineMemOperand::Flags::MOVolatile) 944 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 945 if (Flags & MachineMemOperand::Flags::MONonTemporal) 946 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 947 948 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 949 MIB.addImm(SpvMemOp); 950 } 951 952 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 953 const SPIRVType *ResType, 954 MachineInstr &I) const { 955 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 956 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 957 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 958 .addDef(ResVReg) 959 .addUse(GR.getSPIRVTypeID(ResType)) 960 .addUse(Ptr); 961 if (!I.getNumMemOperands()) { 962 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 963 I.getOpcode() == 964 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 965 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 966 } else { 967 addMemoryOperands(*I.memoperands_begin(), MIB); 968 } 969 return MIB.constrainAllUses(TII, TRI, RBI); 970 } 971 972 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 973 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 974 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 975 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 976 MachineBasicBlock &BB = *I.getParent(); 977 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 978 .addUse(Ptr) 979 .addUse(StoreVal); 980 if (!I.getNumMemOperands()) { 981 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 982 I.getOpcode() == 983 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 984 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 985 } else { 986 addMemoryOperands(*I.memoperands_begin(), MIB); 987 } 988 return MIB.constrainAllUses(TII, TRI, RBI); 989 } 990 991 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 992 const SPIRVType *ResType, 993 MachineInstr &I) const { 994 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 995 report_fatal_error( 996 "llvm.stacksave intrinsic: this instruction requires the following " 997 "SPIR-V extension: SPV_INTEL_variable_length_array", 998 false); 999 MachineBasicBlock &BB = *I.getParent(); 1000 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1001 .addDef(ResVReg) 1002 .addUse(GR.getSPIRVTypeID(ResType)) 1003 .constrainAllUses(TII, TRI, RBI); 1004 } 1005 1006 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1007 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1008 report_fatal_error( 1009 "llvm.stackrestore intrinsic: this instruction requires the following " 1010 "SPIR-V extension: SPV_INTEL_variable_length_array", 1011 false); 1012 if (!I.getOperand(0).isReg()) 1013 return false; 1014 MachineBasicBlock &BB = *I.getParent(); 1015 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1016 .addUse(I.getOperand(0).getReg()) 1017 .constrainAllUses(TII, TRI, RBI); 1018 } 1019 1020 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1021 MachineInstr &I) const { 1022 MachineBasicBlock &BB = *I.getParent(); 1023 Register SrcReg = I.getOperand(1).getReg(); 1024 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1025 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1026 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1027 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1028 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1029 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1030 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1031 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1032 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1033 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1034 Function &CurFunction = GR.CurMF->getFunction(); 1035 Type *LLVMArrTy = 1036 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1037 // Module takes ownership of the global var. 1038 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1039 true, GlobalValue::InternalLinkage, 1040 Constant::getNullValue(LLVMArrTy)); 1041 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1042 GR.add(GV, GR.CurMF, VarReg); 1043 1044 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1045 .addDef(VarReg) 1046 .addUse(GR.getSPIRVTypeID(VarTy)) 1047 .addImm(SPIRV::StorageClass::UniformConstant) 1048 .addUse(Const) 1049 .constrainAllUses(TII, TRI, RBI); 1050 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1051 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1052 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1053 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1054 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 1055 } 1056 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1057 .addUse(I.getOperand(0).getReg()) 1058 .addUse(SrcReg) 1059 .addUse(I.getOperand(2).getReg()); 1060 if (I.getNumMemOperands()) 1061 addMemoryOperands(*I.memoperands_begin(), MIB); 1062 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1063 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1064 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1065 .addUse(MIB->getOperand(0).getReg()); 1066 return Result; 1067 } 1068 1069 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1070 const SPIRVType *ResType, 1071 MachineInstr &I, 1072 unsigned NewOpcode, 1073 unsigned NegateOpcode) const { 1074 assert(I.hasOneMemOperand()); 1075 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1076 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1077 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1078 Register ScopeReg = buildI32Constant(Scope, I); 1079 1080 Register Ptr = I.getOperand(1).getReg(); 1081 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1082 // auto ScSem = 1083 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1084 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1085 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1086 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 1087 1088 bool Result = false; 1089 Register ValueReg = I.getOperand(2).getReg(); 1090 if (NegateOpcode != 0) { 1091 // Translation with negative value operand is requested 1092 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1093 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1094 ValueReg = TmpReg; 1095 } 1096 1097 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1098 .addDef(ResVReg) 1099 .addUse(GR.getSPIRVTypeID(ResType)) 1100 .addUse(Ptr) 1101 .addUse(ScopeReg) 1102 .addUse(MemSemReg) 1103 .addUse(ValueReg) 1104 .constrainAllUses(TII, TRI, RBI); 1105 return Result; 1106 } 1107 1108 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1109 unsigned ArgI = I.getNumOperands() - 1; 1110 Register SrcReg = 1111 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1112 SPIRVType *DefType = 1113 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1114 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1115 report_fatal_error( 1116 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1117 1118 SPIRVType *ScalarType = 1119 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1120 MachineBasicBlock &BB = *I.getParent(); 1121 bool Res = false; 1122 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1123 Register ResVReg = I.getOperand(i).getReg(); 1124 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1125 if (!ResType) { 1126 // There was no "assign type" actions, let's fix this now 1127 ResType = ScalarType; 1128 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1129 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1130 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1131 } 1132 auto MIB = 1133 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1134 .addDef(ResVReg) 1135 .addUse(GR.getSPIRVTypeID(ResType)) 1136 .addUse(SrcReg) 1137 .addImm(static_cast<int64_t>(i)); 1138 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1139 } 1140 return Res; 1141 } 1142 1143 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1144 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1145 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1146 Register MemSemReg = buildI32Constant(MemSem, I); 1147 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1148 uint32_t Scope = static_cast<uint32_t>( 1149 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1150 Register ScopeReg = buildI32Constant(Scope, I); 1151 MachineBasicBlock &BB = *I.getParent(); 1152 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1153 .addUse(ScopeReg) 1154 .addUse(MemSemReg) 1155 .constrainAllUses(TII, TRI, RBI); 1156 } 1157 1158 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1159 const SPIRVType *ResType, 1160 MachineInstr &I, 1161 unsigned Opcode) const { 1162 Type *ResTy = nullptr; 1163 StringRef ResName; 1164 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1165 report_fatal_error( 1166 "Not enough info to select the arithmetic with overflow instruction"); 1167 if (!ResTy || !ResTy->isStructTy()) 1168 report_fatal_error("Expect struct type result for the arithmetic " 1169 "with overflow instruction"); 1170 // "Result Type must be from OpTypeStruct. The struct must have two members, 1171 // and the two members must be the same type." 1172 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1173 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1174 // Build SPIR-V types and constant(s) if needed. 1175 MachineIRBuilder MIRBuilder(I); 1176 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1177 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1178 assert(I.getNumDefs() > 1 && "Not enought operands"); 1179 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1180 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1181 if (N > 1) 1182 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1183 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1184 Register ZeroReg = buildZerosVal(ResType, I); 1185 // A new virtual register to store the result struct. 1186 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1187 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1188 // Build the result name if needed. 1189 if (ResName.size() > 0) 1190 buildOpName(StructVReg, ResName, MIRBuilder); 1191 // Build the arithmetic with overflow instruction. 1192 MachineBasicBlock &BB = *I.getParent(); 1193 auto MIB = 1194 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1195 .addDef(StructVReg) 1196 .addUse(GR.getSPIRVTypeID(StructType)); 1197 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1198 MIB.addUse(I.getOperand(i).getReg()); 1199 bool Status = MIB.constrainAllUses(TII, TRI, RBI); 1200 // Build instructions to extract fields of the instruction's result. 1201 // A new virtual register to store the higher part of the result struct. 1202 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1203 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1204 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1205 auto MIB = 1206 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1207 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1208 .addUse(GR.getSPIRVTypeID(ResType)) 1209 .addUse(StructVReg) 1210 .addImm(i); 1211 Status &= MIB.constrainAllUses(TII, TRI, RBI); 1212 } 1213 // Build boolean value from the higher part. 1214 Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1215 .addDef(I.getOperand(1).getReg()) 1216 .addUse(BoolTypeReg) 1217 .addUse(HigherVReg) 1218 .addUse(ZeroReg) 1219 .constrainAllUses(TII, TRI, RBI); 1220 return Status; 1221 } 1222 1223 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1224 const SPIRVType *ResType, 1225 MachineInstr &I) const { 1226 Register ScopeReg; 1227 Register MemSemEqReg; 1228 Register MemSemNeqReg; 1229 Register Ptr = I.getOperand(2).getReg(); 1230 if (!isa<GIntrinsic>(I)) { 1231 assert(I.hasOneMemOperand()); 1232 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1233 unsigned Scope = static_cast<uint32_t>(getMemScope( 1234 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1235 ScopeReg = buildI32Constant(Scope, I); 1236 1237 unsigned ScSem = static_cast<uint32_t>( 1238 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1239 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1240 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1241 MemSemEqReg = buildI32Constant(MemSemEq, I); 1242 AtomicOrdering FO = MemOp->getFailureOrdering(); 1243 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1244 MemSemNeqReg = 1245 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 1246 } else { 1247 ScopeReg = I.getOperand(5).getReg(); 1248 MemSemEqReg = I.getOperand(6).getReg(); 1249 MemSemNeqReg = I.getOperand(7).getReg(); 1250 } 1251 1252 Register Cmp = I.getOperand(3).getReg(); 1253 Register Val = I.getOperand(4).getReg(); 1254 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1255 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1256 const DebugLoc &DL = I.getDebugLoc(); 1257 bool Result = 1258 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1259 .addDef(ACmpRes) 1260 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1261 .addUse(Ptr) 1262 .addUse(ScopeReg) 1263 .addUse(MemSemEqReg) 1264 .addUse(MemSemNeqReg) 1265 .addUse(Val) 1266 .addUse(Cmp) 1267 .constrainAllUses(TII, TRI, RBI); 1268 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1269 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1270 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1271 .addDef(CmpSuccReg) 1272 .addUse(GR.getSPIRVTypeID(BoolTy)) 1273 .addUse(ACmpRes) 1274 .addUse(Cmp) 1275 .constrainAllUses(TII, TRI, RBI); 1276 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1277 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1278 .addDef(TmpReg) 1279 .addUse(GR.getSPIRVTypeID(ResType)) 1280 .addUse(ACmpRes) 1281 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1282 .addImm(0) 1283 .constrainAllUses(TII, TRI, RBI); 1284 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1285 .addDef(ResVReg) 1286 .addUse(GR.getSPIRVTypeID(ResType)) 1287 .addUse(CmpSuccReg) 1288 .addUse(TmpReg) 1289 .addImm(1) 1290 .constrainAllUses(TII, TRI, RBI); 1291 return Result; 1292 } 1293 1294 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1295 switch (SC) { 1296 case SPIRV::StorageClass::Workgroup: 1297 case SPIRV::StorageClass::CrossWorkgroup: 1298 case SPIRV::StorageClass::Function: 1299 return true; 1300 default: 1301 return false; 1302 } 1303 } 1304 1305 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1306 switch (SC) { 1307 case SPIRV::StorageClass::DeviceOnlyINTEL: 1308 case SPIRV::StorageClass::HostOnlyINTEL: 1309 return true; 1310 default: 1311 return false; 1312 } 1313 } 1314 1315 // Returns true ResVReg is referred only from global vars and OpName's. 1316 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1317 bool IsGRef = false; 1318 bool IsAllowedRefs = 1319 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1320 [&IsGRef](auto const &It) { 1321 unsigned Opcode = It.getOpcode(); 1322 if (Opcode == SPIRV::OpConstantComposite || 1323 Opcode == SPIRV::OpVariable || 1324 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1325 return IsGRef = true; 1326 return Opcode == SPIRV::OpName; 1327 }); 1328 return IsAllowedRefs && IsGRef; 1329 } 1330 1331 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1332 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1333 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1334 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1335 } 1336 1337 MachineInstrBuilder 1338 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1339 Register Src, Register DestType, 1340 uint32_t Opcode) const { 1341 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1342 TII.get(SPIRV::OpSpecConstantOp)) 1343 .addDef(Dest) 1344 .addUse(DestType) 1345 .addImm(Opcode) 1346 .addUse(Src); 1347 } 1348 1349 MachineInstrBuilder 1350 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1351 SPIRVType *SrcPtrTy) const { 1352 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1353 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1354 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1355 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1356 SPIRV::StorageClass::Generic), 1357 GR.getPointerSize())); 1358 MachineFunction *MF = I.getParent()->getParent(); 1359 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1360 MachineInstrBuilder MIB = buildSpecConstantOp( 1361 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1362 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1363 GR.add(MIB.getInstr(), MF, Tmp); 1364 return MIB; 1365 } 1366 1367 // In SPIR-V address space casting can only happen to and from the Generic 1368 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1369 // pointers to and from Generic pointers. As such, we can convert e.g. from 1370 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1371 // other combinations can only be done by a bitcast, and are probably not safe. 1372 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1373 const SPIRVType *ResType, 1374 MachineInstr &I) const { 1375 MachineBasicBlock &BB = *I.getParent(); 1376 const DebugLoc &DL = I.getDebugLoc(); 1377 1378 Register SrcPtr = I.getOperand(1).getReg(); 1379 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1380 1381 // don't generate a cast for a null that may be represented by OpTypeInt 1382 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1383 ResType->getOpcode() != SPIRV::OpTypePointer) 1384 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1385 .addDef(ResVReg) 1386 .addUse(SrcPtr) 1387 .constrainAllUses(TII, TRI, RBI); 1388 1389 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1390 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1391 1392 if (isASCastInGVar(MRI, ResVReg)) { 1393 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1394 // are expressed by OpSpecConstantOp with an Opcode. 1395 // TODO: maybe insert a check whether the Kernel capability was declared and 1396 // so PtrCastToGeneric/GenericCastToPtr are available. 1397 unsigned SpecOpcode = 1398 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1399 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1400 : (SrcSC == SPIRV::StorageClass::Generic && 1401 isGenericCastablePtr(DstSC) 1402 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1403 : 0); 1404 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1405 // correct value of ResType and use general i8* instead. Maybe this should 1406 // be addressed in the emit-intrinsic step to infer a correct 1407 // OpConstantComposite type. 1408 if (SpecOpcode) { 1409 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1410 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1411 .constrainAllUses(TII, TRI, RBI); 1412 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1413 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1414 return MIB.constrainAllUses(TII, TRI, RBI) && 1415 buildSpecConstantOp( 1416 I, ResVReg, MIB->getOperand(0).getReg(), 1417 getUcharPtrTypeReg(I, DstSC), 1418 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1419 .constrainAllUses(TII, TRI, RBI); 1420 } 1421 } 1422 1423 // don't generate a cast between identical storage classes 1424 if (SrcSC == DstSC) 1425 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1426 .addDef(ResVReg) 1427 .addUse(SrcPtr) 1428 .constrainAllUses(TII, TRI, RBI); 1429 1430 // Casting from an eligible pointer to Generic. 1431 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1432 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1433 // Casting from Generic to an eligible pointer. 1434 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1435 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1436 // Casting between 2 eligible pointers using Generic as an intermediary. 1437 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1438 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1439 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1440 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1441 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1442 .addDef(Tmp) 1443 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1444 .addUse(SrcPtr) 1445 .constrainAllUses(TII, TRI, RBI); 1446 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1447 .addDef(ResVReg) 1448 .addUse(GR.getSPIRVTypeID(ResType)) 1449 .addUse(Tmp) 1450 .constrainAllUses(TII, TRI, RBI); 1451 } 1452 1453 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1454 // be applied 1455 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1456 return selectUnOp(ResVReg, ResType, I, 1457 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1458 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1459 return selectUnOp(ResVReg, ResType, I, 1460 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1461 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1462 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1463 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1464 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1465 1466 // Bitcast for pointers requires that the address spaces must match 1467 return false; 1468 } 1469 1470 static unsigned getFCmpOpcode(unsigned PredNum) { 1471 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1472 switch (Pred) { 1473 case CmpInst::FCMP_OEQ: 1474 return SPIRV::OpFOrdEqual; 1475 case CmpInst::FCMP_OGE: 1476 return SPIRV::OpFOrdGreaterThanEqual; 1477 case CmpInst::FCMP_OGT: 1478 return SPIRV::OpFOrdGreaterThan; 1479 case CmpInst::FCMP_OLE: 1480 return SPIRV::OpFOrdLessThanEqual; 1481 case CmpInst::FCMP_OLT: 1482 return SPIRV::OpFOrdLessThan; 1483 case CmpInst::FCMP_ONE: 1484 return SPIRV::OpFOrdNotEqual; 1485 case CmpInst::FCMP_ORD: 1486 return SPIRV::OpOrdered; 1487 case CmpInst::FCMP_UEQ: 1488 return SPIRV::OpFUnordEqual; 1489 case CmpInst::FCMP_UGE: 1490 return SPIRV::OpFUnordGreaterThanEqual; 1491 case CmpInst::FCMP_UGT: 1492 return SPIRV::OpFUnordGreaterThan; 1493 case CmpInst::FCMP_ULE: 1494 return SPIRV::OpFUnordLessThanEqual; 1495 case CmpInst::FCMP_ULT: 1496 return SPIRV::OpFUnordLessThan; 1497 case CmpInst::FCMP_UNE: 1498 return SPIRV::OpFUnordNotEqual; 1499 case CmpInst::FCMP_UNO: 1500 return SPIRV::OpUnordered; 1501 default: 1502 llvm_unreachable("Unknown predicate type for FCmp"); 1503 } 1504 } 1505 1506 static unsigned getICmpOpcode(unsigned PredNum) { 1507 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1508 switch (Pred) { 1509 case CmpInst::ICMP_EQ: 1510 return SPIRV::OpIEqual; 1511 case CmpInst::ICMP_NE: 1512 return SPIRV::OpINotEqual; 1513 case CmpInst::ICMP_SGE: 1514 return SPIRV::OpSGreaterThanEqual; 1515 case CmpInst::ICMP_SGT: 1516 return SPIRV::OpSGreaterThan; 1517 case CmpInst::ICMP_SLE: 1518 return SPIRV::OpSLessThanEqual; 1519 case CmpInst::ICMP_SLT: 1520 return SPIRV::OpSLessThan; 1521 case CmpInst::ICMP_UGE: 1522 return SPIRV::OpUGreaterThanEqual; 1523 case CmpInst::ICMP_UGT: 1524 return SPIRV::OpUGreaterThan; 1525 case CmpInst::ICMP_ULE: 1526 return SPIRV::OpULessThanEqual; 1527 case CmpInst::ICMP_ULT: 1528 return SPIRV::OpULessThan; 1529 default: 1530 llvm_unreachable("Unknown predicate type for ICmp"); 1531 } 1532 } 1533 1534 static unsigned getPtrCmpOpcode(unsigned Pred) { 1535 switch (static_cast<CmpInst::Predicate>(Pred)) { 1536 case CmpInst::ICMP_EQ: 1537 return SPIRV::OpPtrEqual; 1538 case CmpInst::ICMP_NE: 1539 return SPIRV::OpPtrNotEqual; 1540 default: 1541 llvm_unreachable("Unknown predicate type for pointer comparison"); 1542 } 1543 } 1544 1545 // Return the logical operation, or abort if none exists. 1546 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1547 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1548 switch (Pred) { 1549 case CmpInst::ICMP_EQ: 1550 return SPIRV::OpLogicalEqual; 1551 case CmpInst::ICMP_NE: 1552 return SPIRV::OpLogicalNotEqual; 1553 default: 1554 llvm_unreachable("Unknown predicate type for Bool comparison"); 1555 } 1556 } 1557 1558 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1559 if (!LLVMFloatTy) 1560 return APFloat::getZero(APFloat::IEEEsingle()); 1561 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1562 case Type::HalfTyID: 1563 return APFloat::getZero(APFloat::IEEEhalf()); 1564 default: 1565 case Type::FloatTyID: 1566 return APFloat::getZero(APFloat::IEEEsingle()); 1567 case Type::DoubleTyID: 1568 return APFloat::getZero(APFloat::IEEEdouble()); 1569 } 1570 } 1571 1572 static APFloat getOneFP(const Type *LLVMFloatTy) { 1573 if (!LLVMFloatTy) 1574 return APFloat::getOne(APFloat::IEEEsingle()); 1575 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1576 case Type::HalfTyID: 1577 return APFloat::getOne(APFloat::IEEEhalf()); 1578 default: 1579 case Type::FloatTyID: 1580 return APFloat::getOne(APFloat::IEEEsingle()); 1581 case Type::DoubleTyID: 1582 return APFloat::getOne(APFloat::IEEEdouble()); 1583 } 1584 } 1585 1586 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1587 const SPIRVType *ResType, 1588 MachineInstr &I, 1589 unsigned OpAnyOrAll) const { 1590 assert(I.getNumOperands() == 3); 1591 assert(I.getOperand(2).isReg()); 1592 MachineBasicBlock &BB = *I.getParent(); 1593 Register InputRegister = I.getOperand(2).getReg(); 1594 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1595 1596 if (!InputType) 1597 report_fatal_error("Input Type could not be determined."); 1598 1599 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1600 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1601 if (IsBoolTy && !IsVectorTy) { 1602 assert(ResVReg == I.getOperand(0).getReg()); 1603 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1604 TII.get(TargetOpcode::COPY)) 1605 .addDef(ResVReg) 1606 .addUse(InputRegister) 1607 .constrainAllUses(TII, TRI, RBI); 1608 } 1609 1610 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1611 unsigned SpirvNotEqualId = 1612 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1613 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1614 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1615 Register NotEqualReg = ResVReg; 1616 1617 if (IsVectorTy) { 1618 NotEqualReg = IsBoolTy ? InputRegister 1619 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1620 const unsigned NumElts = InputType->getOperand(2).getImm(); 1621 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1622 } 1623 1624 if (!IsBoolTy) { 1625 Register ConstZeroReg = 1626 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1627 1628 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1629 .addDef(NotEqualReg) 1630 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1631 .addUse(InputRegister) 1632 .addUse(ConstZeroReg) 1633 .constrainAllUses(TII, TRI, RBI); 1634 } 1635 1636 if (!IsVectorTy) 1637 return true; 1638 1639 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1640 .addDef(ResVReg) 1641 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1642 .addUse(NotEqualReg) 1643 .constrainAllUses(TII, TRI, RBI); 1644 } 1645 1646 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1647 const SPIRVType *ResType, 1648 MachineInstr &I) const { 1649 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1650 } 1651 1652 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1653 const SPIRVType *ResType, 1654 MachineInstr &I) const { 1655 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1656 } 1657 1658 // Select the OpDot instruction for the given float dot 1659 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1660 const SPIRVType *ResType, 1661 MachineInstr &I) const { 1662 assert(I.getNumOperands() == 4); 1663 assert(I.getOperand(2).isReg()); 1664 assert(I.getOperand(3).isReg()); 1665 1666 [[maybe_unused]] SPIRVType *VecType = 1667 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1668 1669 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1670 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1671 "dot product requires a vector of at least 2 components"); 1672 1673 [[maybe_unused]] SPIRVType *EltType = 1674 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1675 1676 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1677 1678 MachineBasicBlock &BB = *I.getParent(); 1679 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1680 .addDef(ResVReg) 1681 .addUse(GR.getSPIRVTypeID(ResType)) 1682 .addUse(I.getOperand(2).getReg()) 1683 .addUse(I.getOperand(3).getReg()) 1684 .constrainAllUses(TII, TRI, RBI); 1685 } 1686 1687 // Since pre-1.6 SPIRV has no integer dot implementation, 1688 // expand by piecewise multiplying and adding the results 1689 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1690 const SPIRVType *ResType, 1691 MachineInstr &I) const { 1692 assert(I.getNumOperands() == 4); 1693 assert(I.getOperand(2).isReg()); 1694 assert(I.getOperand(3).isReg()); 1695 MachineBasicBlock &BB = *I.getParent(); 1696 1697 // Multiply the vectors, then sum the results 1698 Register Vec0 = I.getOperand(2).getReg(); 1699 Register Vec1 = I.getOperand(3).getReg(); 1700 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1701 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1702 1703 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1704 .addDef(TmpVec) 1705 .addUse(GR.getSPIRVTypeID(VecType)) 1706 .addUse(Vec0) 1707 .addUse(Vec1) 1708 .constrainAllUses(TII, TRI, RBI); 1709 1710 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1711 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1712 "dot product requires a vector of at least 2 components"); 1713 1714 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1715 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1716 .addDef(Res) 1717 .addUse(GR.getSPIRVTypeID(ResType)) 1718 .addUse(TmpVec) 1719 .addImm(0) 1720 .constrainAllUses(TII, TRI, RBI); 1721 1722 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1723 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1724 1725 Result &= 1726 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1727 .addDef(Elt) 1728 .addUse(GR.getSPIRVTypeID(ResType)) 1729 .addUse(TmpVec) 1730 .addImm(i) 1731 .constrainAllUses(TII, TRI, RBI); 1732 1733 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1734 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1735 : ResVReg; 1736 1737 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1738 .addDef(Sum) 1739 .addUse(GR.getSPIRVTypeID(ResType)) 1740 .addUse(Res) 1741 .addUse(Elt) 1742 .constrainAllUses(TII, TRI, RBI); 1743 Res = Sum; 1744 } 1745 1746 return Result; 1747 } 1748 1749 template <bool Signed> 1750 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1751 const SPIRVType *ResType, 1752 MachineInstr &I) const { 1753 assert(I.getNumOperands() == 5); 1754 assert(I.getOperand(2).isReg()); 1755 assert(I.getOperand(3).isReg()); 1756 assert(I.getOperand(4).isReg()); 1757 MachineBasicBlock &BB = *I.getParent(); 1758 1759 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1760 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1761 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1762 .addDef(Dot) 1763 .addUse(GR.getSPIRVTypeID(ResType)) 1764 .addUse(I.getOperand(2).getReg()) 1765 .addUse(I.getOperand(3).getReg()) 1766 .constrainAllUses(TII, TRI, RBI); 1767 1768 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1769 .addDef(ResVReg) 1770 .addUse(GR.getSPIRVTypeID(ResType)) 1771 .addUse(Dot) 1772 .addUse(I.getOperand(4).getReg()) 1773 .constrainAllUses(TII, TRI, RBI); 1774 1775 return Result; 1776 } 1777 1778 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1779 // extract the elements of the packed inputs, multiply them and add the result 1780 // to the accumulator. 1781 template <bool Signed> 1782 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1783 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1784 assert(I.getNumOperands() == 5); 1785 assert(I.getOperand(2).isReg()); 1786 assert(I.getOperand(3).isReg()); 1787 assert(I.getOperand(4).isReg()); 1788 MachineBasicBlock &BB = *I.getParent(); 1789 1790 bool Result = false; 1791 1792 // Acc = C 1793 Register Acc = I.getOperand(4).getReg(); 1794 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1795 auto ExtractOp = 1796 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1797 1798 // Extract the i8 element, multiply and add it to the accumulator 1799 for (unsigned i = 0; i < 4; i++) { 1800 // A[i] 1801 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1802 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1803 .addDef(AElt) 1804 .addUse(GR.getSPIRVTypeID(ResType)) 1805 .addUse(I.getOperand(2).getReg()) 1806 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1807 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1808 .constrainAllUses(TII, TRI, RBI); 1809 1810 // B[i] 1811 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1812 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1813 .addDef(BElt) 1814 .addUse(GR.getSPIRVTypeID(ResType)) 1815 .addUse(I.getOperand(3).getReg()) 1816 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1817 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1818 .constrainAllUses(TII, TRI, RBI); 1819 1820 // A[i] * B[i] 1821 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1822 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1823 .addDef(Mul) 1824 .addUse(GR.getSPIRVTypeID(ResType)) 1825 .addUse(AElt) 1826 .addUse(BElt) 1827 .constrainAllUses(TII, TRI, RBI); 1828 1829 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1830 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1831 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1832 .addDef(MaskMul) 1833 .addUse(GR.getSPIRVTypeID(ResType)) 1834 .addUse(Mul) 1835 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1836 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1837 .constrainAllUses(TII, TRI, RBI); 1838 1839 // Acc = Acc + A[i] * B[i] 1840 Register Sum = 1841 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1842 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1843 .addDef(Sum) 1844 .addUse(GR.getSPIRVTypeID(ResType)) 1845 .addUse(Acc) 1846 .addUse(MaskMul) 1847 .constrainAllUses(TII, TRI, RBI); 1848 1849 Acc = Sum; 1850 } 1851 1852 return Result; 1853 } 1854 1855 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1856 /// does not have a saturate builtin. 1857 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1858 const SPIRVType *ResType, 1859 MachineInstr &I) const { 1860 assert(I.getNumOperands() == 3); 1861 assert(I.getOperand(2).isReg()); 1862 MachineBasicBlock &BB = *I.getParent(); 1863 Register VZero = buildZerosValF(ResType, I); 1864 Register VOne = buildOnesValF(ResType, I); 1865 1866 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1867 .addDef(ResVReg) 1868 .addUse(GR.getSPIRVTypeID(ResType)) 1869 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1870 .addImm(GL::FClamp) 1871 .addUse(I.getOperand(2).getReg()) 1872 .addUse(VZero) 1873 .addUse(VOne) 1874 .constrainAllUses(TII, TRI, RBI); 1875 } 1876 1877 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1878 const SPIRVType *ResType, 1879 MachineInstr &I) const { 1880 assert(I.getNumOperands() == 3); 1881 assert(I.getOperand(2).isReg()); 1882 MachineBasicBlock &BB = *I.getParent(); 1883 Register InputRegister = I.getOperand(2).getReg(); 1884 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1885 auto &DL = I.getDebugLoc(); 1886 1887 if (!InputType) 1888 report_fatal_error("Input Type could not be determined."); 1889 1890 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1891 1892 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1893 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1894 1895 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1896 1897 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1898 Register SignReg = NeedsConversion 1899 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1900 : ResVReg; 1901 1902 bool Result = 1903 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1904 .addDef(SignReg) 1905 .addUse(GR.getSPIRVTypeID(InputType)) 1906 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1907 .addImm(SignOpcode) 1908 .addUse(InputRegister) 1909 .constrainAllUses(TII, TRI, RBI); 1910 1911 if (NeedsConversion) { 1912 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1913 Result |= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1914 .addDef(ResVReg) 1915 .addUse(GR.getSPIRVTypeID(ResType)) 1916 .addUse(SignReg) 1917 .constrainAllUses(TII, TRI, RBI); 1918 } 1919 1920 return Result; 1921 } 1922 1923 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 1924 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1925 assert(I.getNumOperands() == 3); 1926 assert(I.getOperand(2).isReg()); 1927 MachineBasicBlock &BB = *I.getParent(); 1928 1929 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1930 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 1931 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 1932 1933 bool Result = 1934 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformBallot)) 1935 .addDef(BallotReg) 1936 .addUse(GR.getSPIRVTypeID(BallotType)) 1937 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1938 .addUse(I.getOperand(2).getReg()) 1939 .constrainAllUses(TII, TRI, RBI); 1940 1941 Result &= 1942 BuildMI(BB, I, I.getDebugLoc(), 1943 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 1944 .addDef(ResVReg) 1945 .addUse(GR.getSPIRVTypeID(ResType)) 1946 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1947 .addImm(SPIRV::GroupOperation::Reduce) 1948 .addUse(BallotReg) 1949 .constrainAllUses(TII, TRI, RBI); 1950 1951 return Result; 1952 } 1953 1954 bool SPIRVInstructionSelector::selectWaveReadLaneAt(Register ResVReg, 1955 const SPIRVType *ResType, 1956 MachineInstr &I) const { 1957 assert(I.getNumOperands() == 4); 1958 assert(I.getOperand(2).isReg()); 1959 assert(I.getOperand(3).isReg()); 1960 MachineBasicBlock &BB = *I.getParent(); 1961 1962 // IntTy is used to define the execution scope, set to 3 to denote a 1963 // cross-lane interaction equivalent to a SPIR-V subgroup. 1964 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1965 return BuildMI(BB, I, I.getDebugLoc(), 1966 TII.get(SPIRV::OpGroupNonUniformShuffle)) 1967 .addDef(ResVReg) 1968 .addUse(GR.getSPIRVTypeID(ResType)) 1969 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)) 1970 .addUse(I.getOperand(2).getReg()) 1971 .addUse(I.getOperand(3).getReg()); 1972 } 1973 1974 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1975 const SPIRVType *ResType, 1976 MachineInstr &I) const { 1977 MachineBasicBlock &BB = *I.getParent(); 1978 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1979 .addDef(ResVReg) 1980 .addUse(GR.getSPIRVTypeID(ResType)) 1981 .addUse(I.getOperand(1).getReg()) 1982 .constrainAllUses(TII, TRI, RBI); 1983 } 1984 1985 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1986 const SPIRVType *ResType, 1987 MachineInstr &I) const { 1988 // There is no way to implement `freeze` correctly without support on SPIR-V 1989 // standard side, but we may at least address a simple (static) case when 1990 // undef/poison value presence is obvious. The main benefit of even 1991 // incomplete `freeze` support is preventing of translation from crashing due 1992 // to lack of support on legalization and instruction selection steps. 1993 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1994 return false; 1995 Register OpReg = I.getOperand(1).getReg(); 1996 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1997 Register Reg; 1998 switch (Def->getOpcode()) { 1999 case SPIRV::ASSIGN_TYPE: 2000 if (MachineInstr *AssignToDef = 2001 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2002 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2003 Reg = Def->getOperand(2).getReg(); 2004 } 2005 break; 2006 case SPIRV::OpUndef: 2007 Reg = Def->getOperand(1).getReg(); 2008 break; 2009 } 2010 unsigned DestOpCode; 2011 if (Reg.isValid()) { 2012 DestOpCode = SPIRV::OpConstantNull; 2013 } else { 2014 DestOpCode = TargetOpcode::COPY; 2015 Reg = OpReg; 2016 } 2017 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2018 .addDef(I.getOperand(0).getReg()) 2019 .addUse(Reg) 2020 .constrainAllUses(TII, TRI, RBI); 2021 } 2022 return false; 2023 } 2024 2025 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2026 const SPIRVType *ResType) { 2027 Register OpReg = ResType->getOperand(2).getReg(); 2028 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2029 if (!OpDef) 2030 return 0; 2031 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2032 OpDef->getOperand(1).isReg()) { 2033 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2034 OpDef = RefDef; 2035 } 2036 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2037 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2038 : 0; 2039 return N; 2040 } 2041 2042 // Return true if the type represents a constant register 2043 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2044 SmallPtrSet<SPIRVType *, 4> &Visited) { 2045 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2046 OpDef->getOperand(1).isReg()) { 2047 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2048 OpDef = RefDef; 2049 } 2050 2051 if (Visited.contains(OpDef)) 2052 return true; 2053 Visited.insert(OpDef); 2054 2055 unsigned Opcode = OpDef->getOpcode(); 2056 switch (Opcode) { 2057 case TargetOpcode::G_CONSTANT: 2058 case TargetOpcode::G_FCONSTANT: 2059 return true; 2060 case TargetOpcode::G_INTRINSIC: 2061 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2062 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2063 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2064 Intrinsic::spv_const_composite; 2065 case TargetOpcode::G_BUILD_VECTOR: 2066 case TargetOpcode::G_SPLAT_VECTOR: { 2067 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2068 i++) { 2069 SPIRVType *OpNestedDef = 2070 OpDef->getOperand(i).isReg() 2071 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2072 : nullptr; 2073 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2074 return false; 2075 } 2076 return true; 2077 } 2078 } 2079 return false; 2080 } 2081 2082 // Return true if the virtual register represents a constant 2083 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2084 SmallPtrSet<SPIRVType *, 4> Visited; 2085 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2086 return isConstReg(MRI, OpDef, Visited); 2087 return false; 2088 } 2089 2090 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2091 const SPIRVType *ResType, 2092 MachineInstr &I) const { 2093 unsigned N = 0; 2094 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2095 N = GR.getScalarOrVectorComponentCount(ResType); 2096 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2097 N = getArrayComponentCount(MRI, ResType); 2098 else 2099 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2100 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2101 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2102 2103 // check if we may construct a constant vector 2104 bool IsConst = true; 2105 for (unsigned i = I.getNumExplicitDefs(); 2106 i < I.getNumExplicitOperands() && IsConst; ++i) 2107 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2108 IsConst = false; 2109 2110 if (!IsConst && N < 2) 2111 report_fatal_error( 2112 "There must be at least two constituent operands in a vector"); 2113 2114 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2115 TII.get(IsConst ? SPIRV::OpConstantComposite 2116 : SPIRV::OpCompositeConstruct)) 2117 .addDef(ResVReg) 2118 .addUse(GR.getSPIRVTypeID(ResType)); 2119 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2120 MIB.addUse(I.getOperand(i).getReg()); 2121 return MIB.constrainAllUses(TII, TRI, RBI); 2122 } 2123 2124 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2125 const SPIRVType *ResType, 2126 MachineInstr &I) const { 2127 unsigned N = 0; 2128 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2129 N = GR.getScalarOrVectorComponentCount(ResType); 2130 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2131 N = getArrayComponentCount(MRI, ResType); 2132 else 2133 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2134 2135 unsigned OpIdx = I.getNumExplicitDefs(); 2136 if (!I.getOperand(OpIdx).isReg()) 2137 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2138 2139 // check if we may construct a constant vector 2140 Register OpReg = I.getOperand(OpIdx).getReg(); 2141 bool IsConst = isConstReg(MRI, OpReg); 2142 2143 if (!IsConst && N < 2) 2144 report_fatal_error( 2145 "There must be at least two constituent operands in a vector"); 2146 2147 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2148 TII.get(IsConst ? SPIRV::OpConstantComposite 2149 : SPIRV::OpCompositeConstruct)) 2150 .addDef(ResVReg) 2151 .addUse(GR.getSPIRVTypeID(ResType)); 2152 for (unsigned i = 0; i < N; ++i) 2153 MIB.addUse(OpReg); 2154 return MIB.constrainAllUses(TII, TRI, RBI); 2155 } 2156 2157 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2158 const SPIRVType *ResType, 2159 unsigned CmpOpc, 2160 MachineInstr &I) const { 2161 Register Cmp0 = I.getOperand(2).getReg(); 2162 Register Cmp1 = I.getOperand(3).getReg(); 2163 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2164 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2165 "CMP operands should have the same type"); 2166 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2167 .addDef(ResVReg) 2168 .addUse(GR.getSPIRVTypeID(ResType)) 2169 .addUse(Cmp0) 2170 .addUse(Cmp1) 2171 .constrainAllUses(TII, TRI, RBI); 2172 } 2173 2174 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2175 const SPIRVType *ResType, 2176 MachineInstr &I) const { 2177 auto Pred = I.getOperand(1).getPredicate(); 2178 unsigned CmpOpc; 2179 2180 Register CmpOperand = I.getOperand(2).getReg(); 2181 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2182 CmpOpc = getPtrCmpOpcode(Pred); 2183 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2184 CmpOpc = getBoolCmpOpcode(Pred); 2185 else 2186 CmpOpc = getICmpOpcode(Pred); 2187 return selectCmp(ResVReg, ResType, CmpOpc, I); 2188 } 2189 2190 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2191 const MachineInstr &I, 2192 int OpIdx) const { 2193 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2194 "Expected G_FCONSTANT"); 2195 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2196 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2197 } 2198 2199 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2200 const MachineInstr &I, 2201 int OpIdx) const { 2202 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2203 "Expected G_CONSTANT"); 2204 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2205 } 2206 2207 Register 2208 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2209 const SPIRVType *ResType) const { 2210 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2211 const SPIRVType *SpvI32Ty = 2212 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2213 // Find a constant in DT or build a new one. 2214 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2215 Register NewReg = GR.find(ConstInt, GR.CurMF); 2216 if (!NewReg.isValid()) { 2217 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2218 GR.add(ConstInt, GR.CurMF, NewReg); 2219 MachineInstr *MI; 2220 MachineBasicBlock &BB = *I.getParent(); 2221 if (Val == 0) { 2222 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2223 .addDef(NewReg) 2224 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2225 } else { 2226 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2227 .addDef(NewReg) 2228 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2229 .addImm(APInt(32, Val).getZExtValue()); 2230 } 2231 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2232 } 2233 return NewReg; 2234 } 2235 2236 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2237 const SPIRVType *ResType, 2238 MachineInstr &I) const { 2239 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2240 return selectCmp(ResVReg, ResType, CmpOp, I); 2241 } 2242 2243 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2244 MachineInstr &I) const { 2245 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2246 bool ZeroAsNull = STI.isOpenCLEnv(); 2247 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2248 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2249 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2250 } 2251 2252 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2253 MachineInstr &I) const { 2254 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2255 bool ZeroAsNull = STI.isOpenCLEnv(); 2256 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2257 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2258 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2259 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2260 } 2261 2262 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2263 MachineInstr &I) const { 2264 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2265 bool ZeroAsNull = STI.isOpenCLEnv(); 2266 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2267 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2268 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2269 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2270 } 2271 2272 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2273 const SPIRVType *ResType, 2274 MachineInstr &I) const { 2275 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2276 APInt One = 2277 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2278 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2279 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2280 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2281 } 2282 2283 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2284 const SPIRVType *ResType, 2285 MachineInstr &I, 2286 bool IsSigned) const { 2287 // To extend a bool, we need to use OpSelect between constants. 2288 Register ZeroReg = buildZerosVal(ResType, I); 2289 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2290 bool IsScalarBool = 2291 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2292 unsigned Opcode = 2293 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2294 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2295 .addDef(ResVReg) 2296 .addUse(GR.getSPIRVTypeID(ResType)) 2297 .addUse(I.getOperand(1).getReg()) 2298 .addUse(OneReg) 2299 .addUse(ZeroReg) 2300 .constrainAllUses(TII, TRI, RBI); 2301 } 2302 2303 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2304 const SPIRVType *ResType, 2305 MachineInstr &I, bool IsSigned, 2306 unsigned Opcode) const { 2307 Register SrcReg = I.getOperand(1).getReg(); 2308 // We can convert bool value directly to float type without OpConvert*ToF, 2309 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2310 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2311 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2312 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2313 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2314 const unsigned NumElts = ResType->getOperand(2).getImm(); 2315 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2316 } 2317 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2318 selectSelect(SrcReg, TmpType, I, false); 2319 } 2320 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2321 } 2322 2323 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2324 const SPIRVType *ResType, 2325 MachineInstr &I, bool IsSigned) const { 2326 Register SrcReg = I.getOperand(1).getReg(); 2327 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2328 return selectSelect(ResVReg, ResType, I, IsSigned); 2329 2330 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2331 if (SrcType == ResType) { 2332 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2333 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2334 if (DstRC != SrcRC && SrcRC) 2335 MRI->setRegClass(ResVReg, SrcRC); 2336 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2337 TII.get(TargetOpcode::COPY)) 2338 .addDef(ResVReg) 2339 .addUse(SrcReg) 2340 .constrainAllUses(TII, TRI, RBI); 2341 } 2342 2343 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2344 return selectUnOp(ResVReg, ResType, I, Opcode); 2345 } 2346 2347 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2348 Register ResVReg, 2349 MachineInstr &I, 2350 const SPIRVType *IntTy, 2351 const SPIRVType *BoolTy) const { 2352 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2353 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2354 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2355 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2356 Register Zero = buildZerosVal(IntTy, I); 2357 Register One = buildOnesVal(false, IntTy, I); 2358 MachineBasicBlock &BB = *I.getParent(); 2359 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2360 .addDef(BitIntReg) 2361 .addUse(GR.getSPIRVTypeID(IntTy)) 2362 .addUse(IntReg) 2363 .addUse(One) 2364 .constrainAllUses(TII, TRI, RBI); 2365 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2366 .addDef(ResVReg) 2367 .addUse(GR.getSPIRVTypeID(BoolTy)) 2368 .addUse(BitIntReg) 2369 .addUse(Zero) 2370 .constrainAllUses(TII, TRI, RBI); 2371 } 2372 2373 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2374 const SPIRVType *ResType, 2375 MachineInstr &I) const { 2376 Register IntReg = I.getOperand(1).getReg(); 2377 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2378 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2379 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2380 if (ArgType == ResType) { 2381 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2382 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2383 if (DstRC != SrcRC && SrcRC) 2384 MRI->setRegClass(ResVReg, SrcRC); 2385 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2386 TII.get(TargetOpcode::COPY)) 2387 .addDef(ResVReg) 2388 .addUse(IntReg) 2389 .constrainAllUses(TII, TRI, RBI); 2390 } 2391 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2392 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2393 return selectUnOp(ResVReg, ResType, I, Opcode); 2394 } 2395 2396 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2397 const SPIRVType *ResType, 2398 const APInt &Imm, 2399 MachineInstr &I) const { 2400 unsigned TyOpcode = ResType->getOpcode(); 2401 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2402 MachineBasicBlock &BB = *I.getParent(); 2403 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2404 Imm.isZero()) 2405 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2406 .addDef(ResVReg) 2407 .addUse(GR.getSPIRVTypeID(ResType)) 2408 .constrainAllUses(TII, TRI, RBI); 2409 if (TyOpcode == SPIRV::OpTypeInt) { 2410 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2411 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2412 if (Reg == ResVReg) 2413 return true; 2414 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2415 .addDef(ResVReg) 2416 .addUse(Reg) 2417 .constrainAllUses(TII, TRI, RBI); 2418 } 2419 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2420 .addDef(ResVReg) 2421 .addUse(GR.getSPIRVTypeID(ResType)); 2422 // <=32-bit integers should be caught by the sdag pattern. 2423 assert(Imm.getBitWidth() > 32); 2424 addNumImm(Imm, MIB); 2425 return MIB.constrainAllUses(TII, TRI, RBI); 2426 } 2427 2428 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2429 const SPIRVType *ResType, 2430 MachineInstr &I) const { 2431 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2432 .addDef(ResVReg) 2433 .addUse(GR.getSPIRVTypeID(ResType)) 2434 .constrainAllUses(TII, TRI, RBI); 2435 } 2436 2437 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2438 assert(MO.isReg()); 2439 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2440 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2441 assert(TypeInst->getOperand(1).isReg()); 2442 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2443 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2444 } 2445 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2446 } 2447 2448 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2449 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2450 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2451 return TypeInst->getOperand(2).getImm(); 2452 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2453 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2454 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2455 } 2456 2457 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2458 const SPIRVType *ResType, 2459 MachineInstr &I) const { 2460 MachineBasicBlock &BB = *I.getParent(); 2461 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2462 .addDef(ResVReg) 2463 .addUse(GR.getSPIRVTypeID(ResType)) 2464 // object to insert 2465 .addUse(I.getOperand(3).getReg()) 2466 // composite to insert into 2467 .addUse(I.getOperand(2).getReg()); 2468 for (unsigned i = 4; i < I.getNumOperands(); i++) 2469 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2470 return MIB.constrainAllUses(TII, TRI, RBI); 2471 } 2472 2473 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2474 const SPIRVType *ResType, 2475 MachineInstr &I) const { 2476 MachineBasicBlock &BB = *I.getParent(); 2477 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2478 .addDef(ResVReg) 2479 .addUse(GR.getSPIRVTypeID(ResType)) 2480 .addUse(I.getOperand(2).getReg()); 2481 for (unsigned i = 3; i < I.getNumOperands(); i++) 2482 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2483 return MIB.constrainAllUses(TII, TRI, RBI); 2484 } 2485 2486 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2487 const SPIRVType *ResType, 2488 MachineInstr &I) const { 2489 if (isImm(I.getOperand(4), MRI)) 2490 return selectInsertVal(ResVReg, ResType, I); 2491 MachineBasicBlock &BB = *I.getParent(); 2492 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2493 .addDef(ResVReg) 2494 .addUse(GR.getSPIRVTypeID(ResType)) 2495 .addUse(I.getOperand(2).getReg()) 2496 .addUse(I.getOperand(3).getReg()) 2497 .addUse(I.getOperand(4).getReg()) 2498 .constrainAllUses(TII, TRI, RBI); 2499 } 2500 2501 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2502 const SPIRVType *ResType, 2503 MachineInstr &I) const { 2504 if (isImm(I.getOperand(3), MRI)) 2505 return selectExtractVal(ResVReg, ResType, I); 2506 MachineBasicBlock &BB = *I.getParent(); 2507 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2508 .addDef(ResVReg) 2509 .addUse(GR.getSPIRVTypeID(ResType)) 2510 .addUse(I.getOperand(2).getReg()) 2511 .addUse(I.getOperand(3).getReg()) 2512 .constrainAllUses(TII, TRI, RBI); 2513 } 2514 2515 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2516 const SPIRVType *ResType, 2517 MachineInstr &I) const { 2518 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2519 2520 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2521 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2522 // we have to use Op[InBounds]AccessChain. 2523 const unsigned Opcode = STI.isVulkanEnv() 2524 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2525 : SPIRV::OpAccessChain) 2526 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2527 : SPIRV::OpPtrAccessChain); 2528 2529 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2530 .addDef(ResVReg) 2531 .addUse(GR.getSPIRVTypeID(ResType)) 2532 // Object to get a pointer to. 2533 .addUse(I.getOperand(3).getReg()); 2534 // Adding indices. 2535 const unsigned StartingIndex = 2536 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2537 ? 5 2538 : 4; 2539 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2540 Res.addUse(I.getOperand(i).getReg()); 2541 return Res.constrainAllUses(TII, TRI, RBI); 2542 } 2543 2544 // Maybe wrap a value into OpSpecConstantOp 2545 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2546 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2547 bool Result = true; 2548 unsigned Lim = I.getNumExplicitOperands(); 2549 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2550 Register OpReg = I.getOperand(i).getReg(); 2551 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2552 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2553 SmallPtrSet<SPIRVType *, 4> Visited; 2554 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2555 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2556 GR.isAggregateType(OpType)) { 2557 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2558 // by selectAddrSpaceCast() 2559 CompositeArgs.push_back(OpReg); 2560 continue; 2561 } 2562 MachineFunction *MF = I.getMF(); 2563 Register WrapReg = GR.find(OpDefine, MF); 2564 if (WrapReg.isValid()) { 2565 CompositeArgs.push_back(WrapReg); 2566 continue; 2567 } 2568 // Create a new register for the wrapper 2569 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2570 GR.add(OpDefine, MF, WrapReg); 2571 CompositeArgs.push_back(WrapReg); 2572 // Decorate the wrapper register and generate a new instruction 2573 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2574 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2575 MachineBasicBlock &BB = *I.getParent(); 2576 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2577 .addDef(WrapReg) 2578 .addUse(GR.getSPIRVTypeID(OpType)) 2579 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2580 .addUse(OpReg) 2581 .constrainAllUses(TII, TRI, RBI); 2582 if (!Result) 2583 break; 2584 } 2585 return Result; 2586 } 2587 2588 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2589 const SPIRVType *ResType, 2590 MachineInstr &I) const { 2591 MachineBasicBlock &BB = *I.getParent(); 2592 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2593 switch (IID) { 2594 case Intrinsic::spv_load: 2595 return selectLoad(ResVReg, ResType, I); 2596 case Intrinsic::spv_store: 2597 return selectStore(I); 2598 case Intrinsic::spv_extractv: 2599 return selectExtractVal(ResVReg, ResType, I); 2600 case Intrinsic::spv_insertv: 2601 return selectInsertVal(ResVReg, ResType, I); 2602 case Intrinsic::spv_extractelt: 2603 return selectExtractElt(ResVReg, ResType, I); 2604 case Intrinsic::spv_insertelt: 2605 return selectInsertElt(ResVReg, ResType, I); 2606 case Intrinsic::spv_gep: 2607 return selectGEP(ResVReg, ResType, I); 2608 case Intrinsic::spv_unref_global: 2609 case Intrinsic::spv_init_global: { 2610 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2611 MachineInstr *Init = I.getNumExplicitOperands() > 2 2612 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2613 : nullptr; 2614 assert(MI); 2615 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2616 } 2617 case Intrinsic::spv_undef: { 2618 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2619 .addDef(ResVReg) 2620 .addUse(GR.getSPIRVTypeID(ResType)); 2621 return MIB.constrainAllUses(TII, TRI, RBI); 2622 } 2623 case Intrinsic::spv_const_composite: { 2624 // If no values are attached, the composite is null constant. 2625 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2626 // Select a proper instruction. 2627 unsigned Opcode = SPIRV::OpConstantNull; 2628 SmallVector<Register> CompositeArgs; 2629 if (!IsNull) { 2630 Opcode = SPIRV::OpConstantComposite; 2631 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2632 return false; 2633 } 2634 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2635 .addDef(ResVReg) 2636 .addUse(GR.getSPIRVTypeID(ResType)); 2637 // skip type MD node we already used when generated assign.type for this 2638 if (!IsNull) { 2639 for (Register OpReg : CompositeArgs) 2640 MIB.addUse(OpReg); 2641 } 2642 return MIB.constrainAllUses(TII, TRI, RBI); 2643 } 2644 case Intrinsic::spv_assign_name: { 2645 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2646 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2647 for (unsigned i = I.getNumExplicitDefs() + 2; 2648 i < I.getNumExplicitOperands(); ++i) { 2649 MIB.addImm(I.getOperand(i).getImm()); 2650 } 2651 return MIB.constrainAllUses(TII, TRI, RBI); 2652 } 2653 case Intrinsic::spv_switch: { 2654 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2655 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2656 if (I.getOperand(i).isReg()) 2657 MIB.addReg(I.getOperand(i).getReg()); 2658 else if (I.getOperand(i).isCImm()) 2659 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2660 else if (I.getOperand(i).isMBB()) 2661 MIB.addMBB(I.getOperand(i).getMBB()); 2662 else 2663 llvm_unreachable("Unexpected OpSwitch operand"); 2664 } 2665 return MIB.constrainAllUses(TII, TRI, RBI); 2666 } 2667 case Intrinsic::spv_loop_merge: 2668 case Intrinsic::spv_selection_merge: { 2669 const auto Opcode = IID == Intrinsic::spv_selection_merge 2670 ? SPIRV::OpSelectionMerge 2671 : SPIRV::OpLoopMerge; 2672 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2673 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2674 assert(I.getOperand(i).isMBB()); 2675 MIB.addMBB(I.getOperand(i).getMBB()); 2676 } 2677 MIB.addImm(SPIRV::SelectionControl::None); 2678 return MIB.constrainAllUses(TII, TRI, RBI); 2679 } 2680 case Intrinsic::spv_cmpxchg: 2681 return selectAtomicCmpXchg(ResVReg, ResType, I); 2682 case Intrinsic::spv_unreachable: 2683 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 2684 break; 2685 case Intrinsic::spv_alloca: 2686 return selectFrameIndex(ResVReg, ResType, I); 2687 case Intrinsic::spv_alloca_array: 2688 return selectAllocaArray(ResVReg, ResType, I); 2689 case Intrinsic::spv_assume: 2690 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2691 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2692 .addUse(I.getOperand(1).getReg()); 2693 break; 2694 case Intrinsic::spv_expect: 2695 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2696 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2697 .addDef(ResVReg) 2698 .addUse(GR.getSPIRVTypeID(ResType)) 2699 .addUse(I.getOperand(2).getReg()) 2700 .addUse(I.getOperand(3).getReg()); 2701 break; 2702 case Intrinsic::arithmetic_fence: 2703 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2704 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT)) 2705 .addDef(ResVReg) 2706 .addUse(GR.getSPIRVTypeID(ResType)) 2707 .addUse(I.getOperand(2).getReg()); 2708 else 2709 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 2710 .addUse(I.getOperand(2).getReg()); 2711 break; 2712 case Intrinsic::spv_thread_id: 2713 return selectSpvThreadId(ResVReg, ResType, I); 2714 case Intrinsic::spv_fdot: 2715 return selectFloatDot(ResVReg, ResType, I); 2716 case Intrinsic::spv_udot: 2717 case Intrinsic::spv_sdot: 2718 return selectIntegerDot(ResVReg, ResType, I); 2719 case Intrinsic::spv_dot4add_i8packed: 2720 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2721 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2722 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2723 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2724 case Intrinsic::spv_dot4add_u8packed: 2725 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2726 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2727 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2728 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2729 case Intrinsic::spv_all: 2730 return selectAll(ResVReg, ResType, I); 2731 case Intrinsic::spv_any: 2732 return selectAny(ResVReg, ResType, I); 2733 case Intrinsic::spv_cross: 2734 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2735 case Intrinsic::spv_lerp: 2736 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2737 case Intrinsic::spv_length: 2738 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2739 case Intrinsic::spv_degrees: 2740 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2741 case Intrinsic::spv_frac: 2742 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2743 case Intrinsic::spv_normalize: 2744 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2745 case Intrinsic::spv_rsqrt: 2746 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2747 case Intrinsic::spv_sign: 2748 return selectSign(ResVReg, ResType, I); 2749 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2750 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2751 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2752 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2753 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2754 Register MemSemReg = 2755 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2756 Register ScopeReg = buildI32Constant(SPIRV::Scope::Workgroup, I); 2757 MachineBasicBlock &BB = *I.getParent(); 2758 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2759 .addUse(ScopeReg) 2760 .addUse(ScopeReg) 2761 .addUse(MemSemReg) 2762 .constrainAllUses(TII, TRI, RBI); 2763 } break; 2764 case Intrinsic::spv_lifetime_start: 2765 case Intrinsic::spv_lifetime_end: { 2766 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2767 : SPIRV::OpLifetimeStop; 2768 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2769 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2770 if (Size == -1) 2771 Size = 0; 2772 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 2773 } break; 2774 case Intrinsic::spv_saturate: 2775 return selectSaturate(ResVReg, ResType, I); 2776 case Intrinsic::spv_nclamp: 2777 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2778 case Intrinsic::spv_uclamp: 2779 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2780 case Intrinsic::spv_sclamp: 2781 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2782 case Intrinsic::spv_wave_active_countbits: 2783 return selectWaveActiveCountBits(ResVReg, ResType, I); 2784 case Intrinsic::spv_wave_is_first_lane: { 2785 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2786 return BuildMI(BB, I, I.getDebugLoc(), 2787 TII.get(SPIRV::OpGroupNonUniformElect)) 2788 .addDef(ResVReg) 2789 .addUse(GR.getSPIRVTypeID(ResType)) 2790 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2791 } 2792 case Intrinsic::spv_wave_readlane: 2793 return selectWaveReadLaneAt(ResVReg, ResType, I); 2794 case Intrinsic::spv_step: 2795 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2796 case Intrinsic::spv_radians: 2797 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2798 // Discard intrinsics which we do not expect to actually represent code after 2799 // lowering or intrinsics which are not implemented but should not crash when 2800 // found in a customer's LLVM IR input. 2801 case Intrinsic::instrprof_increment: 2802 case Intrinsic::instrprof_increment_step: 2803 case Intrinsic::instrprof_value_profile: 2804 break; 2805 // Discard internal intrinsics. 2806 case Intrinsic::spv_value_md: 2807 break; 2808 case Intrinsic::spv_handle_fromBinding: { 2809 selectHandleFromBinding(ResVReg, ResType, I); 2810 return true; 2811 } 2812 default: { 2813 std::string DiagMsg; 2814 raw_string_ostream OS(DiagMsg); 2815 I.print(OS); 2816 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2817 report_fatal_error(DiagMsg.c_str(), false); 2818 } 2819 } 2820 return true; 2821 } 2822 2823 void SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2824 const SPIRVType *ResType, 2825 MachineInstr &I) const { 2826 2827 uint32_t Set = foldImm(I.getOperand(2), MRI); 2828 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2829 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2830 Register IndexReg = I.getOperand(5).getReg(); 2831 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2832 2833 MachineIRBuilder MIRBuilder(I); 2834 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2835 IndexReg, IsNonUniform, MIRBuilder); 2836 2837 if (IsNonUniform) 2838 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2839 2840 // TODO: For now we assume the resource is an image, which needs to be 2841 // loaded to get the handle. That will not be true for storage buffers. 2842 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2843 .addDef(ResVReg) 2844 .addUse(GR.getSPIRVTypeID(ResType)) 2845 .addUse(VarReg); 2846 } 2847 2848 Register SPIRVInstructionSelector::buildPointerToResource( 2849 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 2850 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 2851 MachineIRBuilder MIRBuilder) const { 2852 if (ArraySize == 1) 2853 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 2854 MIRBuilder); 2855 2856 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 2857 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 2858 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 2859 VarType, Set, Binding, MIRBuilder); 2860 2861 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 2862 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 2863 2864 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2865 if (IsNonUniform) { 2866 // It is unclear which value needs to be marked an non-uniform, so both 2867 // the index and the access changed are decorated as non-uniform. 2868 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2869 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2870 } 2871 2872 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 2873 .addDef(AcReg) 2874 .addUse(GR.getSPIRVTypeID(ResPointerType)) 2875 .addUse(VarReg) 2876 .addUse(IndexReg); 2877 2878 return AcReg; 2879 } 2880 2881 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 2882 const SPIRVType *ResType, 2883 MachineInstr &I, 2884 bool IsSigned) const { 2885 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2886 // zero or sign extend 2887 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2888 bool Result = 2889 selectUnOpWithSrc(ExtReg, ResType, I, I.getOperand(2).getReg(), Opcode); 2890 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 2891 } 2892 2893 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 2894 const SPIRVType *ResType, 2895 MachineInstr &I, 2896 Register SrcReg, 2897 bool IsSigned) const { 2898 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 2899 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2900 .addDef(ResVReg) 2901 .addUse(GR.getSPIRVTypeID(ResType)) 2902 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2903 .addImm(Opcode) 2904 .addUse(SrcReg) 2905 .constrainAllUses(TII, TRI, RBI); 2906 } 2907 2908 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 2909 const SPIRVType *ResType, 2910 MachineInstr &I, 2911 bool IsSigned) const { 2912 Register OpReg = I.getOperand(2).getReg(); 2913 // 1. split our int64 into 2 pieces using a bitcast 2914 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 2915 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 2916 MachineIRBuilder MIRBuilder(I); 2917 SPIRVType *postCastT = 2918 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 2919 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 2920 bool Result = 2921 selectUnOpWithSrc(bitcastReg, postCastT, I, OpReg, SPIRV::OpBitcast); 2922 2923 // 2. call firstbithigh 2924 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 2925 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 2926 2927 // 3. split result vector into high bits and low bits 2928 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2929 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2930 2931 bool ZeroAsNull = STI.isOpenCLEnv(); 2932 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 2933 if (isScalarRes) { 2934 // if scalar do a vector extract 2935 Result &= selectNAryOpWithSrcs( 2936 HighReg, ResType, I, 2937 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 2938 SPIRV::OpVectorExtractDynamic); 2939 Result &= selectNAryOpWithSrcs( 2940 LowReg, ResType, I, 2941 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 2942 SPIRV::OpVectorExtractDynamic); 2943 } else { // vector case do a shufflevector 2944 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2945 TII.get(SPIRV::OpVectorShuffle)) 2946 .addDef(HighReg) 2947 .addUse(GR.getSPIRVTypeID(ResType)) 2948 .addUse(FBHReg) 2949 .addUse(FBHReg); 2950 // ^^ this vector will not be selected from; could be empty 2951 unsigned j; 2952 for (j = 0; j < count * 2; j += 2) { 2953 MIB.addImm(j); 2954 } 2955 Result &= MIB.constrainAllUses(TII, TRI, RBI); 2956 2957 // get low bits 2958 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2959 TII.get(SPIRV::OpVectorShuffle)) 2960 .addDef(LowReg) 2961 .addUse(GR.getSPIRVTypeID(ResType)) 2962 .addUse(FBHReg) 2963 .addUse(FBHReg); 2964 // ^^ this vector will not be selected from; could be empty 2965 for (j = 1; j < count * 2; j += 2) { 2966 MIB.addImm(j); 2967 } 2968 Result &= MIB.constrainAllUses(TII, TRI, RBI); 2969 } 2970 2971 // 4. check if result of each top 32 bits is == -1 2972 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 2973 Register NegOneReg; 2974 Register Reg0; 2975 Register Reg32; 2976 unsigned selectOp; 2977 unsigned addOp; 2978 if (isScalarRes) { 2979 NegOneReg = 2980 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 2981 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2982 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 2983 selectOp = SPIRV::OpSelectSISCond; 2984 addOp = SPIRV::OpIAddS; 2985 } else { 2986 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 2987 NegOneReg = 2988 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 2989 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 2990 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 2991 selectOp = SPIRV::OpSelectVIVCond; 2992 addOp = SPIRV::OpIAddV; 2993 } 2994 2995 // check if the high bits are == -1; true if -1 2996 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 2997 Result &= selectNAryOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 2998 SPIRV::OpIEqual); 2999 3000 // Select low bits if true in BReg, otherwise high bits 3001 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3002 Result &= selectNAryOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, 3003 selectOp); 3004 3005 // Add 32 for high bits, 0 for low bits 3006 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3007 Result &= 3008 selectNAryOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3009 3010 return Result && 3011 selectNAryOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3012 } 3013 3014 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3015 const SPIRVType *ResType, 3016 MachineInstr &I, 3017 bool IsSigned) const { 3018 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3019 Register OpReg = I.getOperand(2).getReg(); 3020 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3021 3022 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3023 case 16: 3024 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3025 case 32: 3026 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3027 case 64: 3028 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3029 default: 3030 report_fatal_error( 3031 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3032 } 3033 } 3034 3035 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3036 const SPIRVType *ResType, 3037 MachineInstr &I) const { 3038 // there was an allocation size parameter to the allocation instruction 3039 // that is not 1 3040 MachineBasicBlock &BB = *I.getParent(); 3041 return BuildMI(BB, I, I.getDebugLoc(), 3042 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3043 .addDef(ResVReg) 3044 .addUse(GR.getSPIRVTypeID(ResType)) 3045 .addUse(I.getOperand(2).getReg()) 3046 .constrainAllUses(TII, TRI, RBI); 3047 } 3048 3049 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3050 const SPIRVType *ResType, 3051 MachineInstr &I) const { 3052 // Change order of instructions if needed: all OpVariable instructions in a 3053 // function must be the first instructions in the first block 3054 MachineFunction *MF = I.getParent()->getParent(); 3055 MachineBasicBlock *MBB = &MF->front(); 3056 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 3057 bool IsHeader = false; 3058 unsigned Opcode; 3059 for (; It != E && It != I; ++It) { 3060 Opcode = It->getOpcode(); 3061 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 3062 IsHeader = true; 3063 } else if (IsHeader && 3064 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 3065 ++It; 3066 break; 3067 } 3068 } 3069 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 3070 .addDef(ResVReg) 3071 .addUse(GR.getSPIRVTypeID(ResType)) 3072 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3073 .constrainAllUses(TII, TRI, RBI); 3074 } 3075 3076 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3077 // InstructionSelector walks backwards through the instructions. We can use 3078 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3079 // first, so can generate an OpBranchConditional here. If there is no 3080 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3081 const MachineInstr *PrevI = I.getPrevNode(); 3082 MachineBasicBlock &MBB = *I.getParent(); 3083 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3084 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3085 .addUse(PrevI->getOperand(0).getReg()) 3086 .addMBB(PrevI->getOperand(1).getMBB()) 3087 .addMBB(I.getOperand(0).getMBB()) 3088 .constrainAllUses(TII, TRI, RBI); 3089 } 3090 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3091 .addMBB(I.getOperand(0).getMBB()) 3092 .constrainAllUses(TII, TRI, RBI); 3093 } 3094 3095 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3096 // InstructionSelector walks backwards through the instructions. For an 3097 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3098 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3099 // generate the OpBranchConditional in selectBranch above. 3100 // 3101 // If an OpBranchConditional has been generated, we simply return, as the work 3102 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3103 // implicit fallthrough to the next basic block, so we need to create an 3104 // OpBranchConditional with an explicit "false" argument pointing to the next 3105 // basic block that LLVM would fall through to. 3106 const MachineInstr *NextI = I.getNextNode(); 3107 // Check if this has already been successfully selected. 3108 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3109 return true; 3110 // Must be relying on implicit block fallthrough, so generate an 3111 // OpBranchConditional with the "next" basic block as the "false" target. 3112 MachineBasicBlock &MBB = *I.getParent(); 3113 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3114 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3115 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3116 .addUse(I.getOperand(0).getReg()) 3117 .addMBB(I.getOperand(1).getMBB()) 3118 .addMBB(NextMBB) 3119 .constrainAllUses(TII, TRI, RBI); 3120 } 3121 3122 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3123 const SPIRVType *ResType, 3124 MachineInstr &I) const { 3125 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3126 .addDef(ResVReg) 3127 .addUse(GR.getSPIRVTypeID(ResType)); 3128 const unsigned NumOps = I.getNumOperands(); 3129 for (unsigned i = 1; i < NumOps; i += 2) { 3130 MIB.addUse(I.getOperand(i + 0).getReg()); 3131 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3132 } 3133 return MIB.constrainAllUses(TII, TRI, RBI); 3134 } 3135 3136 bool SPIRVInstructionSelector::selectGlobalValue( 3137 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3138 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3139 MachineIRBuilder MIRBuilder(I); 3140 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3141 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3142 SPIRVType *PointerBaseType; 3143 if (GVType->isArrayTy()) { 3144 SPIRVType *ArrayElementType = 3145 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3146 SPIRV::AccessQualifier::ReadWrite, false); 3147 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3148 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3149 } else { 3150 PointerBaseType = GR.getOrCreateSPIRVType( 3151 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3152 } 3153 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3154 PointerBaseType, I, TII, 3155 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3156 3157 std::string GlobalIdent; 3158 if (!GV->hasName()) { 3159 unsigned &ID = UnnamedGlobalIDs[GV]; 3160 if (ID == 0) 3161 ID = UnnamedGlobalIDs.size(); 3162 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3163 } else { 3164 GlobalIdent = GV->getGlobalIdentifier(); 3165 } 3166 3167 // Behaviour of functions as operands depends on availability of the 3168 // corresponding extension (SPV_INTEL_function_pointers): 3169 // - If there is an extension to operate with functions as operands: 3170 // We create a proper constant operand and evaluate a correct type for a 3171 // function pointer. 3172 // - Without the required extension: 3173 // We have functions as operands in tests with blocks of instruction e.g. in 3174 // transcoding/global_block.ll. These operands are not used and should be 3175 // substituted by zero constants. Their type is expected to be always 3176 // OpTypePointer Function %uchar. 3177 if (isa<Function>(GV)) { 3178 const Constant *ConstVal = GV; 3179 MachineBasicBlock &BB = *I.getParent(); 3180 Register NewReg = GR.find(ConstVal, GR.CurMF); 3181 if (!NewReg.isValid()) { 3182 Register NewReg = ResVReg; 3183 GR.add(ConstVal, GR.CurMF, NewReg); 3184 const Function *GVFun = 3185 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3186 ? dyn_cast<Function>(GV) 3187 : nullptr; 3188 if (GVFun) { 3189 // References to a function via function pointers generate virtual 3190 // registers without a definition. We will resolve it later, during 3191 // module analysis stage. 3192 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3193 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3194 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3195 MachineInstrBuilder MB = 3196 BuildMI(BB, I, I.getDebugLoc(), 3197 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3198 .addDef(NewReg) 3199 .addUse(GR.getSPIRVTypeID(ResType)) 3200 .addUse(FuncVReg); 3201 // mapping the function pointer to the used Function 3202 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3203 return MB.constrainAllUses(TII, TRI, RBI); 3204 } 3205 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3206 .addDef(NewReg) 3207 .addUse(GR.getSPIRVTypeID(ResType)) 3208 .constrainAllUses(TII, TRI, RBI); 3209 } 3210 assert(NewReg != ResVReg); 3211 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 3212 .addDef(ResVReg) 3213 .addUse(NewReg) 3214 .constrainAllUses(TII, TRI, RBI); 3215 } 3216 auto GlobalVar = cast<GlobalVariable>(GV); 3217 assert(GlobalVar->getName() != "llvm.global.annotations"); 3218 3219 bool HasInit = GlobalVar->hasInitializer() && 3220 !isa<UndefValue>(GlobalVar->getInitializer()); 3221 // Skip empty declaration for GVs with initilaizers till we get the decl with 3222 // passed initializer. 3223 if (HasInit && !Init) 3224 return true; 3225 3226 unsigned AddrSpace = GV->getAddressSpace(); 3227 SPIRV::StorageClass::StorageClass Storage = 3228 addressSpaceToStorageClass(AddrSpace, STI); 3229 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 3230 Storage != SPIRV::StorageClass::Function; 3231 SPIRV::LinkageType::LinkageType LnkType = 3232 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3233 ? SPIRV::LinkageType::Import 3234 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3235 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3236 ? SPIRV::LinkageType::LinkOnceODR 3237 : SPIRV::LinkageType::Export); 3238 3239 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 3240 Storage, Init, GlobalVar->isConstant(), 3241 HasLnkTy, LnkType, MIRBuilder, true); 3242 return Reg.isValid(); 3243 } 3244 3245 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3246 const SPIRVType *ResType, 3247 MachineInstr &I) const { 3248 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3249 return selectExtInst(ResVReg, ResType, I, CL::log10); 3250 } 3251 3252 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3253 // is implemented as: 3254 // log10(x) = log2(x) * (1 / log2(10)) 3255 // = log2(x) * 0.30103 3256 3257 MachineIRBuilder MIRBuilder(I); 3258 MachineBasicBlock &BB = *I.getParent(); 3259 3260 // Build log2(x). 3261 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3262 bool Result = 3263 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3264 .addDef(VarReg) 3265 .addUse(GR.getSPIRVTypeID(ResType)) 3266 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3267 .addImm(GL::Log2) 3268 .add(I.getOperand(1)) 3269 .constrainAllUses(TII, TRI, RBI); 3270 3271 // Build 0.30103. 3272 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3273 ResType->getOpcode() == SPIRV::OpTypeFloat); 3274 // TODO: Add matrix implementation once supported by the HLSL frontend. 3275 const SPIRVType *SpirvScalarType = 3276 ResType->getOpcode() == SPIRV::OpTypeVector 3277 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3278 : ResType; 3279 Register ScaleReg = 3280 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3281 3282 // Multiply log2(x) by 0.30103 to get log10(x) result. 3283 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3284 ? SPIRV::OpVectorTimesScalar 3285 : SPIRV::OpFMulS; 3286 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3287 .addDef(ResVReg) 3288 .addUse(GR.getSPIRVTypeID(ResType)) 3289 .addUse(VarReg) 3290 .addUse(ScaleReg) 3291 .constrainAllUses(TII, TRI, RBI); 3292 3293 return Result; 3294 } 3295 3296 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3297 const SPIRVType *ResType, 3298 MachineInstr &I) const { 3299 // DX intrinsic: @llvm.dx.thread.id(i32) 3300 // ID Name Description 3301 // 93 ThreadId reads the thread ID 3302 3303 MachineIRBuilder MIRBuilder(I); 3304 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3305 const SPIRVType *Vec3Ty = 3306 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3307 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3308 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3309 3310 // Create new register for GlobalInvocationID builtin variable. 3311 Register NewRegister = 3312 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3313 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3314 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3315 3316 // Build GlobalInvocationID global variable with the necessary decorations. 3317 Register Variable = GR.buildGlobalVariable( 3318 NewRegister, PtrType, 3319 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3320 SPIRV::StorageClass::Input, nullptr, true, true, 3321 SPIRV::LinkageType::Import, MIRBuilder, false); 3322 3323 // Create new register for loading value. 3324 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3325 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3326 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3327 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3328 3329 // Load v3uint value from the global variable. 3330 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3331 .addDef(LoadedRegister) 3332 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3333 .addUse(Variable); 3334 3335 // Get Thread ID index. Expecting operand is a constant immediate value, 3336 // wrapped in a type assignment. 3337 assert(I.getOperand(2).isReg()); 3338 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3339 3340 // Extract the thread ID from the loaded vector value. 3341 MachineBasicBlock &BB = *I.getParent(); 3342 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3343 .addDef(ResVReg) 3344 .addUse(GR.getSPIRVTypeID(ResType)) 3345 .addUse(LoadedRegister) 3346 .addImm(ThreadId); 3347 return MIB.constrainAllUses(TII, TRI, RBI); 3348 } 3349 3350 namespace llvm { 3351 InstructionSelector * 3352 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3353 const SPIRVSubtarget &Subtarget, 3354 const RegisterBankInfo &RBI) { 3355 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3356 } 3357 } // namespace llvm 3358