1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/Register.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/IR/IntrinsicsSPIRV.h" 34 #include "llvm/Support/Debug.h" 35 36 #define DEBUG_TYPE "spirv-isel" 37 38 using namespace llvm; 39 namespace CL = SPIRV::OpenCLExtInst; 40 namespace GL = SPIRV::GLSLExtInst; 41 42 using ExtInstList = 43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 44 45 namespace { 46 47 #define GET_GLOBALISEL_PREDICATE_BITSET 48 #include "SPIRVGenGlobalISel.inc" 49 #undef GET_GLOBALISEL_PREDICATE_BITSET 50 51 class SPIRVInstructionSelector : public InstructionSelector { 52 const SPIRVSubtarget &STI; 53 const SPIRVInstrInfo &TII; 54 const SPIRVRegisterInfo &TRI; 55 const RegisterBankInfo &RBI; 56 SPIRVGlobalRegistry &GR; 57 MachineRegisterInfo *MRI; 58 MachineFunction *HasVRegsReset = nullptr; 59 60 /// We need to keep track of the number we give to anonymous global values to 61 /// generate the same name every time when this is needed. 62 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 63 64 public: 65 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 66 const SPIRVSubtarget &ST, 67 const RegisterBankInfo &RBI); 68 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 69 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 70 BlockFrequencyInfo *BFI) override; 71 // Common selection code. Instruction-specific selection occurs in spvSelect. 72 bool select(MachineInstr &I) override; 73 static const char *getName() { return DEBUG_TYPE; } 74 75 #define GET_GLOBALISEL_PREDICATES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_PREDICATES_DECL 78 79 #define GET_GLOBALISEL_TEMPORARIES_DECL 80 #include "SPIRVGenGlobalISel.inc" 81 #undef GET_GLOBALISEL_TEMPORARIES_DECL 82 83 private: 84 void resetVRegsType(MachineFunction &MF); 85 86 // tblgen-erated 'select' implementation, used as the initial selector for 87 // the patterns that don't require complex C++. 88 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 89 90 // All instruction-specific selection that didn't happen in "select()". 91 // Is basically a large Switch/Case delegating to all other select method. 92 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I) const; 94 95 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 96 MachineInstr &I, bool IsSigned) const; 97 98 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I, bool IsSigned) const; 100 101 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 102 MachineInstr &I, Register SrcReg, 103 bool IsSigned) const; 104 105 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 106 MachineInstr &I, bool IsSigned) const; 107 108 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 109 const MachineInstr *Init = nullptr) const; 110 111 bool selectNAryOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I, std::vector<Register> SrcRegs, 113 unsigned Opcode) const; 114 115 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 116 MachineInstr &I, Register SrcReg, 117 unsigned Opcode) const; 118 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 119 unsigned Opcode) const; 120 121 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 125 MachineInstr &I) const; 126 bool selectStore(MachineInstr &I) const; 127 128 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 129 MachineInstr &I) const; 130 bool selectStackRestore(MachineInstr &I) const; 131 132 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 133 134 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 135 MachineInstr &I, unsigned NewOpcode, 136 unsigned NegateOpcode = 0) const; 137 138 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 139 MachineInstr &I) const; 140 141 bool selectFence(MachineInstr &I) const; 142 143 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 144 MachineInstr &I) const; 145 146 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 147 MachineInstr &I, unsigned OpType) const; 148 149 bool selectAll(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 152 bool selectAny(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 158 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 163 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 164 unsigned comparisonOpcode, MachineInstr &I) const; 165 bool selectCross(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 168 MachineInstr &I) const; 169 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I) const; 171 172 bool selectSign(Register ResVReg, const SPIRVType *ResType, 173 MachineInstr &I) const; 174 175 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 176 MachineInstr &I) const; 177 178 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 179 MachineInstr &I, unsigned Opcode) const; 180 181 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 182 MachineInstr &I) const; 183 184 template <bool Signed> 185 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 template <bool Signed> 188 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 192 int OpIdx) const; 193 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 194 int OpIdx) const; 195 196 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 197 MachineInstr &I) const; 198 199 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 200 bool IsSigned) const; 201 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 202 bool IsSigned, unsigned Opcode) const; 203 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 204 bool IsSigned) const; 205 206 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 207 MachineInstr &I) const; 208 209 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 210 const SPIRVType *intTy, const SPIRVType *boolTy) const; 211 212 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 215 MachineInstr &I) const; 216 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I) const; 218 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I) const; 220 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I) const; 224 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 227 MachineInstr &I) const; 228 229 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 230 MachineInstr &I) const; 231 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 232 MachineInstr &I) const; 233 234 bool selectBranch(MachineInstr &I) const; 235 bool selectBranchCond(MachineInstr &I) const; 236 237 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 240 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 241 MachineInstr &I, GL::GLSLExtInst GLInst) const; 242 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 243 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 244 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 245 MachineInstr &I, CL::OpenCLExtInst CLInst, 246 GL::GLSLExtInst GLInst) const; 247 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 248 MachineInstr &I, const ExtInstList &ExtInsts) const; 249 250 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 251 MachineInstr &I) const; 252 253 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 254 MachineInstr &I) const; 255 256 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 257 MachineInstr &I) const; 258 259 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 260 MachineInstr &I) const; 261 262 bool selectWaveReadLaneAt(Register ResVReg, const SPIRVType *ResType, 263 MachineInstr &I) const; 264 265 bool selectUnmergeValues(MachineInstr &I) const; 266 267 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 268 MachineInstr &I) const; 269 270 // Utilities 271 std::pair<Register, bool> 272 buildI32Constant(uint32_t Val, MachineInstr &I, 273 const SPIRVType *ResType = nullptr) const; 274 275 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 276 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 277 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 278 MachineInstr &I) const; 279 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 280 281 bool wrapIntoSpecConstantOp(MachineInstr &I, 282 SmallVector<Register> &CompositeArgs) const; 283 284 Register getUcharPtrTypeReg(MachineInstr &I, 285 SPIRV::StorageClass::StorageClass SC) const; 286 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 287 Register Src, Register DestType, 288 uint32_t Opcode) const; 289 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 290 SPIRVType *SrcPtrTy) const; 291 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 292 uint32_t Binding, uint32_t ArraySize, 293 Register IndexReg, bool IsNonUniform, 294 MachineIRBuilder MIRBuilder) const; 295 }; 296 297 } // end anonymous namespace 298 299 #define GET_GLOBALISEL_IMPL 300 #include "SPIRVGenGlobalISel.inc" 301 #undef GET_GLOBALISEL_IMPL 302 303 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 304 const SPIRVSubtarget &ST, 305 const RegisterBankInfo &RBI) 306 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 307 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 308 #define GET_GLOBALISEL_PREDICATES_INIT 309 #include "SPIRVGenGlobalISel.inc" 310 #undef GET_GLOBALISEL_PREDICATES_INIT 311 #define GET_GLOBALISEL_TEMPORARIES_INIT 312 #include "SPIRVGenGlobalISel.inc" 313 #undef GET_GLOBALISEL_TEMPORARIES_INIT 314 { 315 } 316 317 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 318 CodeGenCoverage *CoverageInfo, 319 ProfileSummaryInfo *PSI, 320 BlockFrequencyInfo *BFI) { 321 MRI = &MF.getRegInfo(); 322 GR.setCurrentFunc(MF); 323 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 324 } 325 326 // Ensure that register classes correspond to pattern matching rules. 327 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 328 if (HasVRegsReset == &MF) 329 return; 330 HasVRegsReset = &MF; 331 332 MachineRegisterInfo &MRI = MF.getRegInfo(); 333 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 334 Register Reg = Register::index2VirtReg(I); 335 LLT RegType = MRI.getType(Reg); 336 if (RegType.isScalar()) 337 MRI.setType(Reg, LLT::scalar(64)); 338 else if (RegType.isPointer()) 339 MRI.setType(Reg, LLT::pointer(0, 64)); 340 else if (RegType.isVector()) 341 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 342 } 343 for (const auto &MBB : MF) { 344 for (const auto &MI : MBB) { 345 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 346 continue; 347 Register DstReg = MI.getOperand(0).getReg(); 348 LLT DstType = MRI.getType(DstReg); 349 Register SrcReg = MI.getOperand(1).getReg(); 350 LLT SrcType = MRI.getType(SrcReg); 351 if (DstType != SrcType) 352 MRI.setType(DstReg, MRI.getType(SrcReg)); 353 354 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 355 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 356 if (DstRC != SrcRC && SrcRC) 357 MRI.setRegClass(DstReg, SrcRC); 358 } 359 } 360 } 361 362 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 363 364 // Defined in SPIRVLegalizerInfo.cpp. 365 extern bool isTypeFoldingSupported(unsigned Opcode); 366 367 bool SPIRVInstructionSelector::select(MachineInstr &I) { 368 resetVRegsType(*I.getParent()->getParent()); 369 370 assert(I.getParent() && "Instruction should be in a basic block!"); 371 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 372 373 Register Opcode = I.getOpcode(); 374 // If it's not a GMIR instruction, we've selected it already. 375 if (!isPreISelGenericOpcode(Opcode)) { 376 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 377 Register DstReg = I.getOperand(0).getReg(); 378 Register SrcReg = I.getOperand(1).getReg(); 379 auto *Def = MRI->getVRegDef(SrcReg); 380 if (isTypeFoldingSupported(Def->getOpcode())) { 381 bool Res = selectImpl(I, *CoverageInfo); 382 LLVM_DEBUG({ 383 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 384 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 385 I.print(dbgs()); 386 } 387 }); 388 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 389 if (Res) 390 return Res; 391 } 392 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 393 MRI->replaceRegWith(SrcReg, DstReg); 394 I.removeFromParent(); 395 return true; 396 } else if (I.getNumDefs() == 1) { 397 // Make all vregs 64 bits (for SPIR-V IDs). 398 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 399 } 400 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 401 } 402 403 if (I.getNumOperands() != I.getNumExplicitOperands()) { 404 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 405 return false; 406 } 407 408 // Common code for getting return reg+type, and removing selected instr 409 // from parent occurs here. Instr-specific selection happens in spvSelect(). 410 bool HasDefs = I.getNumDefs() > 0; 411 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 412 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 413 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 414 if (spvSelect(ResVReg, ResType, I)) { 415 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 416 for (unsigned i = 0; i < I.getNumDefs(); ++i) 417 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 418 I.removeFromParent(); 419 return true; 420 } 421 return false; 422 } 423 424 static bool mayApplyGenericSelection(unsigned Opcode) { 425 switch (Opcode) { 426 case TargetOpcode::G_CONSTANT: 427 return false; 428 case TargetOpcode::G_SADDO: 429 case TargetOpcode::G_SSUBO: 430 return true; 431 } 432 return isTypeFoldingSupported(Opcode); 433 } 434 435 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 436 const SPIRVType *ResType, 437 MachineInstr &I) const { 438 const unsigned Opcode = I.getOpcode(); 439 if (mayApplyGenericSelection(Opcode)) 440 return selectImpl(I, *CoverageInfo); 441 switch (Opcode) { 442 case TargetOpcode::G_CONSTANT: 443 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 444 I); 445 case TargetOpcode::G_GLOBAL_VALUE: 446 return selectGlobalValue(ResVReg, I); 447 case TargetOpcode::G_IMPLICIT_DEF: 448 return selectOpUndef(ResVReg, ResType, I); 449 case TargetOpcode::G_FREEZE: 450 return selectFreeze(ResVReg, ResType, I); 451 452 case TargetOpcode::G_INTRINSIC: 453 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 454 case TargetOpcode::G_INTRINSIC_CONVERGENT: 455 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 456 return selectIntrinsic(ResVReg, ResType, I); 457 case TargetOpcode::G_BITREVERSE: 458 return selectBitreverse(ResVReg, ResType, I); 459 460 case TargetOpcode::G_BUILD_VECTOR: 461 return selectBuildVector(ResVReg, ResType, I); 462 case TargetOpcode::G_SPLAT_VECTOR: 463 return selectSplatVector(ResVReg, ResType, I); 464 465 case TargetOpcode::G_SHUFFLE_VECTOR: { 466 MachineBasicBlock &BB = *I.getParent(); 467 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 468 .addDef(ResVReg) 469 .addUse(GR.getSPIRVTypeID(ResType)) 470 .addUse(I.getOperand(1).getReg()) 471 .addUse(I.getOperand(2).getReg()); 472 for (auto V : I.getOperand(3).getShuffleMask()) 473 MIB.addImm(V); 474 return MIB.constrainAllUses(TII, TRI, RBI); 475 } 476 case TargetOpcode::G_MEMMOVE: 477 case TargetOpcode::G_MEMCPY: 478 case TargetOpcode::G_MEMSET: 479 return selectMemOperation(ResVReg, I); 480 481 case TargetOpcode::G_ICMP: 482 return selectICmp(ResVReg, ResType, I); 483 case TargetOpcode::G_FCMP: 484 return selectFCmp(ResVReg, ResType, I); 485 486 case TargetOpcode::G_FRAME_INDEX: 487 return selectFrameIndex(ResVReg, ResType, I); 488 489 case TargetOpcode::G_LOAD: 490 return selectLoad(ResVReg, ResType, I); 491 case TargetOpcode::G_STORE: 492 return selectStore(I); 493 494 case TargetOpcode::G_BR: 495 return selectBranch(I); 496 case TargetOpcode::G_BRCOND: 497 return selectBranchCond(I); 498 499 case TargetOpcode::G_PHI: 500 return selectPhi(ResVReg, ResType, I); 501 502 case TargetOpcode::G_FPTOSI: 503 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 504 case TargetOpcode::G_FPTOUI: 505 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 506 507 case TargetOpcode::G_SITOFP: 508 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 509 case TargetOpcode::G_UITOFP: 510 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 511 512 case TargetOpcode::G_CTPOP: 513 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 514 case TargetOpcode::G_SMIN: 515 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 516 case TargetOpcode::G_UMIN: 517 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 518 519 case TargetOpcode::G_SMAX: 520 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 521 case TargetOpcode::G_UMAX: 522 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 523 524 case TargetOpcode::G_FMA: 525 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 526 527 case TargetOpcode::G_FPOW: 528 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 529 case TargetOpcode::G_FPOWI: 530 return selectExtInst(ResVReg, ResType, I, CL::pown); 531 532 case TargetOpcode::G_FEXP: 533 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 534 case TargetOpcode::G_FEXP2: 535 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 536 537 case TargetOpcode::G_FLOG: 538 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 539 case TargetOpcode::G_FLOG2: 540 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 541 case TargetOpcode::G_FLOG10: 542 return selectLog10(ResVReg, ResType, I); 543 544 case TargetOpcode::G_FABS: 545 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 546 case TargetOpcode::G_ABS: 547 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 548 549 case TargetOpcode::G_FMINNUM: 550 case TargetOpcode::G_FMINIMUM: 551 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 552 case TargetOpcode::G_FMAXNUM: 553 case TargetOpcode::G_FMAXIMUM: 554 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 555 556 case TargetOpcode::G_FCOPYSIGN: 557 return selectExtInst(ResVReg, ResType, I, CL::copysign); 558 559 case TargetOpcode::G_FCEIL: 560 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 561 case TargetOpcode::G_FFLOOR: 562 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 563 564 case TargetOpcode::G_FCOS: 565 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 566 case TargetOpcode::G_FSIN: 567 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 568 case TargetOpcode::G_FTAN: 569 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 570 case TargetOpcode::G_FACOS: 571 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 572 case TargetOpcode::G_FASIN: 573 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 574 case TargetOpcode::G_FATAN: 575 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 576 case TargetOpcode::G_FATAN2: 577 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 578 case TargetOpcode::G_FCOSH: 579 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 580 case TargetOpcode::G_FSINH: 581 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 582 case TargetOpcode::G_FTANH: 583 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 584 585 case TargetOpcode::G_FSQRT: 586 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 587 588 case TargetOpcode::G_CTTZ: 589 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 590 return selectExtInst(ResVReg, ResType, I, CL::ctz); 591 case TargetOpcode::G_CTLZ: 592 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 593 return selectExtInst(ResVReg, ResType, I, CL::clz); 594 595 case TargetOpcode::G_INTRINSIC_ROUND: 596 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 597 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 598 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 599 case TargetOpcode::G_INTRINSIC_TRUNC: 600 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 601 case TargetOpcode::G_FRINT: 602 case TargetOpcode::G_FNEARBYINT: 603 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 604 605 case TargetOpcode::G_SMULH: 606 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 607 case TargetOpcode::G_UMULH: 608 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 609 610 case TargetOpcode::G_SADDSAT: 611 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 612 case TargetOpcode::G_UADDSAT: 613 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 614 case TargetOpcode::G_SSUBSAT: 615 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 616 case TargetOpcode::G_USUBSAT: 617 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 618 619 case TargetOpcode::G_UADDO: 620 return selectOverflowArith(ResVReg, ResType, I, 621 ResType->getOpcode() == SPIRV::OpTypeVector 622 ? SPIRV::OpIAddCarryV 623 : SPIRV::OpIAddCarryS); 624 case TargetOpcode::G_USUBO: 625 return selectOverflowArith(ResVReg, ResType, I, 626 ResType->getOpcode() == SPIRV::OpTypeVector 627 ? SPIRV::OpISubBorrowV 628 : SPIRV::OpISubBorrowS); 629 case TargetOpcode::G_UMULO: 630 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 631 case TargetOpcode::G_SMULO: 632 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 633 634 case TargetOpcode::G_SEXT: 635 return selectExt(ResVReg, ResType, I, true); 636 case TargetOpcode::G_ANYEXT: 637 case TargetOpcode::G_ZEXT: 638 return selectExt(ResVReg, ResType, I, false); 639 case TargetOpcode::G_TRUNC: 640 return selectTrunc(ResVReg, ResType, I); 641 case TargetOpcode::G_FPTRUNC: 642 case TargetOpcode::G_FPEXT: 643 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 644 645 case TargetOpcode::G_PTRTOINT: 646 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 647 case TargetOpcode::G_INTTOPTR: 648 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 649 case TargetOpcode::G_BITCAST: 650 return selectBitcast(ResVReg, ResType, I); 651 case TargetOpcode::G_ADDRSPACE_CAST: 652 return selectAddrSpaceCast(ResVReg, ResType, I); 653 case TargetOpcode::G_PTR_ADD: { 654 // Currently, we get G_PTR_ADD only applied to global variables. 655 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 656 Register GV = I.getOperand(1).getReg(); 657 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 658 (void)II; 659 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 660 (*II).getOpcode() == TargetOpcode::COPY || 661 (*II).getOpcode() == SPIRV::OpVariable) && 662 isImm(I.getOperand(2), MRI)); 663 // It may be the initialization of a global variable. 664 bool IsGVInit = false; 665 for (MachineRegisterInfo::use_instr_iterator 666 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 667 UseEnd = MRI->use_instr_end(); 668 UseIt != UseEnd; UseIt = std::next(UseIt)) { 669 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 670 (*UseIt).getOpcode() == SPIRV::OpVariable) { 671 IsGVInit = true; 672 break; 673 } 674 } 675 MachineBasicBlock &BB = *I.getParent(); 676 if (!IsGVInit) { 677 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 678 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 679 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 680 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 681 // Build a new virtual register that is associated with the required 682 // data type. 683 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 684 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 685 // Having a correctly typed base we are ready to build the actually 686 // required GEP. It may not be a constant though, because all Operands 687 // of OpSpecConstantOp is to originate from other const instructions, 688 // and only the AccessChain named opcodes accept a global OpVariable 689 // instruction. We can't use an AccessChain opcode because of the type 690 // mismatch between result and base types. 691 if (!GR.isBitcastCompatible(ResType, GVType)) 692 report_fatal_error( 693 "incompatible result and operand types in a bitcast"); 694 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 695 MachineInstrBuilder MIB = 696 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 697 .addDef(NewVReg) 698 .addUse(ResTypeReg) 699 .addUse(GV); 700 return MIB.constrainAllUses(TII, TRI, RBI) && 701 BuildMI(BB, I, I.getDebugLoc(), 702 TII.get(STI.isVulkanEnv() 703 ? SPIRV::OpInBoundsAccessChain 704 : SPIRV::OpInBoundsPtrAccessChain)) 705 .addDef(ResVReg) 706 .addUse(ResTypeReg) 707 .addUse(NewVReg) 708 .addUse(I.getOperand(2).getReg()) 709 .constrainAllUses(TII, TRI, RBI); 710 } else { 711 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 712 .addDef(ResVReg) 713 .addUse(GR.getSPIRVTypeID(ResType)) 714 .addImm( 715 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 716 .addUse(GV) 717 .addUse(I.getOperand(2).getReg()) 718 .constrainAllUses(TII, TRI, RBI); 719 } 720 } 721 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 722 // initialize a global variable with a constant expression (e.g., the test 723 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 724 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 725 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 726 .addDef(ResVReg) 727 .addUse(GR.getSPIRVTypeID(ResType)) 728 .addImm(static_cast<uint32_t>( 729 SPIRV::Opcode::InBoundsPtrAccessChain)) 730 .addUse(GV) 731 .addUse(Idx) 732 .addUse(I.getOperand(2).getReg()); 733 return MIB.constrainAllUses(TII, TRI, RBI); 734 } 735 736 case TargetOpcode::G_ATOMICRMW_OR: 737 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 738 case TargetOpcode::G_ATOMICRMW_ADD: 739 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 740 case TargetOpcode::G_ATOMICRMW_AND: 741 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 742 case TargetOpcode::G_ATOMICRMW_MAX: 743 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 744 case TargetOpcode::G_ATOMICRMW_MIN: 745 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 746 case TargetOpcode::G_ATOMICRMW_SUB: 747 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 748 case TargetOpcode::G_ATOMICRMW_XOR: 749 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 750 case TargetOpcode::G_ATOMICRMW_UMAX: 751 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 752 case TargetOpcode::G_ATOMICRMW_UMIN: 753 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 754 case TargetOpcode::G_ATOMICRMW_XCHG: 755 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 756 case TargetOpcode::G_ATOMIC_CMPXCHG: 757 return selectAtomicCmpXchg(ResVReg, ResType, I); 758 759 case TargetOpcode::G_ATOMICRMW_FADD: 760 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 761 case TargetOpcode::G_ATOMICRMW_FSUB: 762 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 763 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 764 SPIRV::OpFNegate); 765 case TargetOpcode::G_ATOMICRMW_FMIN: 766 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 767 case TargetOpcode::G_ATOMICRMW_FMAX: 768 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 769 770 case TargetOpcode::G_FENCE: 771 return selectFence(I); 772 773 case TargetOpcode::G_STACKSAVE: 774 return selectStackSave(ResVReg, ResType, I); 775 case TargetOpcode::G_STACKRESTORE: 776 return selectStackRestore(I); 777 778 case TargetOpcode::G_UNMERGE_VALUES: 779 return selectUnmergeValues(I); 780 781 // Discard gen opcodes for intrinsics which we do not expect to actually 782 // represent code after lowering or intrinsics which are not implemented but 783 // should not crash when found in a customer's LLVM IR input. 784 case TargetOpcode::G_TRAP: 785 case TargetOpcode::G_DEBUGTRAP: 786 case TargetOpcode::G_UBSANTRAP: 787 case TargetOpcode::DBG_LABEL: 788 return true; 789 790 default: 791 return false; 792 } 793 } 794 795 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 796 const SPIRVType *ResType, 797 MachineInstr &I, 798 GL::GLSLExtInst GLInst) const { 799 return selectExtInst(ResVReg, ResType, I, 800 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 801 } 802 803 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 804 const SPIRVType *ResType, 805 MachineInstr &I, 806 CL::OpenCLExtInst CLInst) const { 807 return selectExtInst(ResVReg, ResType, I, 808 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 809 } 810 811 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 812 const SPIRVType *ResType, 813 MachineInstr &I, 814 CL::OpenCLExtInst CLInst, 815 GL::GLSLExtInst GLInst) const { 816 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 817 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 818 return selectExtInst(ResVReg, ResType, I, ExtInsts); 819 } 820 821 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 822 const SPIRVType *ResType, 823 MachineInstr &I, 824 const ExtInstList &Insts) const { 825 826 for (const auto &Ex : Insts) { 827 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 828 uint32_t Opcode = Ex.second; 829 if (STI.canUseExtInstSet(Set)) { 830 MachineBasicBlock &BB = *I.getParent(); 831 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 832 .addDef(ResVReg) 833 .addUse(GR.getSPIRVTypeID(ResType)) 834 .addImm(static_cast<uint32_t>(Set)) 835 .addImm(Opcode); 836 const unsigned NumOps = I.getNumOperands(); 837 unsigned Index = 1; 838 if (Index < NumOps && 839 I.getOperand(Index).getType() == 840 MachineOperand::MachineOperandType::MO_IntrinsicID) 841 Index = 2; 842 for (; Index < NumOps; ++Index) 843 MIB.add(I.getOperand(Index)); 844 return MIB.constrainAllUses(TII, TRI, RBI); 845 } 846 } 847 return false; 848 } 849 850 bool SPIRVInstructionSelector::selectNAryOpWithSrcs(Register ResVReg, 851 const SPIRVType *ResType, 852 MachineInstr &I, 853 std::vector<Register> Srcs, 854 unsigned Opcode) const { 855 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 856 .addDef(ResVReg) 857 .addUse(GR.getSPIRVTypeID(ResType)); 858 for (Register SReg : Srcs) { 859 MIB.addUse(SReg); 860 } 861 return MIB.constrainAllUses(TII, TRI, RBI); 862 } 863 864 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 865 const SPIRVType *ResType, 866 MachineInstr &I, 867 Register SrcReg, 868 unsigned Opcode) const { 869 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 870 .addDef(ResVReg) 871 .addUse(GR.getSPIRVTypeID(ResType)) 872 .addUse(SrcReg) 873 .constrainAllUses(TII, TRI, RBI); 874 } 875 876 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 877 const SPIRVType *ResType, 878 MachineInstr &I, 879 unsigned Opcode) const { 880 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 881 Register SrcReg = I.getOperand(1).getReg(); 882 bool IsGV = false; 883 for (MachineRegisterInfo::def_instr_iterator DefIt = 884 MRI->def_instr_begin(SrcReg); 885 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 886 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 887 IsGV = true; 888 break; 889 } 890 } 891 if (IsGV) { 892 uint32_t SpecOpcode = 0; 893 switch (Opcode) { 894 case SPIRV::OpConvertPtrToU: 895 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 896 break; 897 case SPIRV::OpConvertUToPtr: 898 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 899 break; 900 } 901 if (SpecOpcode) 902 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 903 TII.get(SPIRV::OpSpecConstantOp)) 904 .addDef(ResVReg) 905 .addUse(GR.getSPIRVTypeID(ResType)) 906 .addImm(SpecOpcode) 907 .addUse(SrcReg) 908 .constrainAllUses(TII, TRI, RBI); 909 } 910 } 911 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 912 Opcode); 913 } 914 915 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 916 const SPIRVType *ResType, 917 MachineInstr &I) const { 918 Register OpReg = I.getOperand(1).getReg(); 919 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 920 if (!GR.isBitcastCompatible(ResType, OpType)) 921 report_fatal_error("incompatible result and operand types in a bitcast"); 922 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 923 } 924 925 static void addMemoryOperands(MachineMemOperand *MemOp, 926 MachineInstrBuilder &MIB) { 927 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 928 if (MemOp->isVolatile()) 929 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 930 if (MemOp->isNonTemporal()) 931 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 932 if (MemOp->getAlign().value()) 933 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 934 935 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 936 MIB.addImm(SpvMemOp); 937 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 938 MIB.addImm(MemOp->getAlign().value()); 939 } 940 } 941 942 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 943 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 944 if (Flags & MachineMemOperand::Flags::MOVolatile) 945 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 946 if (Flags & MachineMemOperand::Flags::MONonTemporal) 947 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 948 949 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 950 MIB.addImm(SpvMemOp); 951 } 952 953 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 954 const SPIRVType *ResType, 955 MachineInstr &I) const { 956 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 957 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 958 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 959 .addDef(ResVReg) 960 .addUse(GR.getSPIRVTypeID(ResType)) 961 .addUse(Ptr); 962 if (!I.getNumMemOperands()) { 963 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 964 I.getOpcode() == 965 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 966 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 967 } else { 968 addMemoryOperands(*I.memoperands_begin(), MIB); 969 } 970 return MIB.constrainAllUses(TII, TRI, RBI); 971 } 972 973 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 974 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 975 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 976 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 977 MachineBasicBlock &BB = *I.getParent(); 978 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 979 .addUse(Ptr) 980 .addUse(StoreVal); 981 if (!I.getNumMemOperands()) { 982 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 983 I.getOpcode() == 984 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 985 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 986 } else { 987 addMemoryOperands(*I.memoperands_begin(), MIB); 988 } 989 return MIB.constrainAllUses(TII, TRI, RBI); 990 } 991 992 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 993 const SPIRVType *ResType, 994 MachineInstr &I) const { 995 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 996 report_fatal_error( 997 "llvm.stacksave intrinsic: this instruction requires the following " 998 "SPIR-V extension: SPV_INTEL_variable_length_array", 999 false); 1000 MachineBasicBlock &BB = *I.getParent(); 1001 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1002 .addDef(ResVReg) 1003 .addUse(GR.getSPIRVTypeID(ResType)) 1004 .constrainAllUses(TII, TRI, RBI); 1005 } 1006 1007 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1008 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1009 report_fatal_error( 1010 "llvm.stackrestore intrinsic: this instruction requires the following " 1011 "SPIR-V extension: SPV_INTEL_variable_length_array", 1012 false); 1013 if (!I.getOperand(0).isReg()) 1014 return false; 1015 MachineBasicBlock &BB = *I.getParent(); 1016 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1017 .addUse(I.getOperand(0).getReg()) 1018 .constrainAllUses(TII, TRI, RBI); 1019 } 1020 1021 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1022 MachineInstr &I) const { 1023 MachineBasicBlock &BB = *I.getParent(); 1024 Register SrcReg = I.getOperand(1).getReg(); 1025 bool Result = true; 1026 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1027 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1028 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1029 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1030 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1031 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1032 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1033 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1034 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1035 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1036 Function &CurFunction = GR.CurMF->getFunction(); 1037 Type *LLVMArrTy = 1038 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1039 // Module takes ownership of the global var. 1040 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1041 true, GlobalValue::InternalLinkage, 1042 Constant::getNullValue(LLVMArrTy)); 1043 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1044 GR.add(GV, GR.CurMF, VarReg); 1045 1046 Result &= 1047 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1048 .addDef(VarReg) 1049 .addUse(GR.getSPIRVTypeID(VarTy)) 1050 .addImm(SPIRV::StorageClass::UniformConstant) 1051 .addUse(Const) 1052 .constrainAllUses(TII, TRI, RBI); 1053 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1054 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1055 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1056 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1057 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 1058 } 1059 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1060 .addUse(I.getOperand(0).getReg()) 1061 .addUse(SrcReg) 1062 .addUse(I.getOperand(2).getReg()); 1063 if (I.getNumMemOperands()) 1064 addMemoryOperands(*I.memoperands_begin(), MIB); 1065 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1066 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1067 Result &= 1068 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1069 .addUse(MIB->getOperand(0).getReg()) 1070 .constrainAllUses(TII, TRI, RBI); 1071 return Result; 1072 } 1073 1074 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1075 const SPIRVType *ResType, 1076 MachineInstr &I, 1077 unsigned NewOpcode, 1078 unsigned NegateOpcode) const { 1079 bool Result = true; 1080 assert(I.hasOneMemOperand()); 1081 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1082 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1083 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1084 auto ScopeConstant = buildI32Constant(Scope, I); 1085 Register ScopeReg = ScopeConstant.first; 1086 Result &= ScopeConstant.second; 1087 1088 Register Ptr = I.getOperand(1).getReg(); 1089 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1090 // auto ScSem = 1091 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1092 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1093 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1094 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1095 Register MemSemReg = MemSemConstant.first; 1096 Result &= MemSemConstant.second; 1097 1098 Register ValueReg = I.getOperand(2).getReg(); 1099 if (NegateOpcode != 0) { 1100 // Translation with negative value operand is requested 1101 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1102 Result &= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1103 ValueReg = TmpReg; 1104 } 1105 1106 return Result && 1107 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1108 .addDef(ResVReg) 1109 .addUse(GR.getSPIRVTypeID(ResType)) 1110 .addUse(Ptr) 1111 .addUse(ScopeReg) 1112 .addUse(MemSemReg) 1113 .addUse(ValueReg) 1114 .constrainAllUses(TII, TRI, RBI); 1115 } 1116 1117 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1118 unsigned ArgI = I.getNumOperands() - 1; 1119 Register SrcReg = 1120 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1121 SPIRVType *DefType = 1122 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1123 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1124 report_fatal_error( 1125 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1126 1127 SPIRVType *ScalarType = 1128 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1129 MachineBasicBlock &BB = *I.getParent(); 1130 bool Res = false; 1131 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1132 Register ResVReg = I.getOperand(i).getReg(); 1133 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1134 if (!ResType) { 1135 // There was no "assign type" actions, let's fix this now 1136 ResType = ScalarType; 1137 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1138 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1139 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1140 } 1141 auto MIB = 1142 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1143 .addDef(ResVReg) 1144 .addUse(GR.getSPIRVTypeID(ResType)) 1145 .addUse(SrcReg) 1146 .addImm(static_cast<int64_t>(i)); 1147 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1148 } 1149 return Res; 1150 } 1151 1152 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1153 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1154 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1155 auto MemSemConstant = buildI32Constant(MemSem, I); 1156 Register MemSemReg = MemSemConstant.first; 1157 bool Result = MemSemConstant.second; 1158 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1159 uint32_t Scope = static_cast<uint32_t>( 1160 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1161 auto ScopeConstant = buildI32Constant(Scope, I); 1162 Register ScopeReg = ScopeConstant.first; 1163 Result &= ScopeConstant.second; 1164 MachineBasicBlock &BB = *I.getParent(); 1165 return Result && 1166 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1167 .addUse(ScopeReg) 1168 .addUse(MemSemReg) 1169 .constrainAllUses(TII, TRI, RBI); 1170 } 1171 1172 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1173 const SPIRVType *ResType, 1174 MachineInstr &I, 1175 unsigned Opcode) const { 1176 Type *ResTy = nullptr; 1177 StringRef ResName; 1178 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1179 report_fatal_error( 1180 "Not enough info to select the arithmetic with overflow instruction"); 1181 if (!ResTy || !ResTy->isStructTy()) 1182 report_fatal_error("Expect struct type result for the arithmetic " 1183 "with overflow instruction"); 1184 // "Result Type must be from OpTypeStruct. The struct must have two members, 1185 // and the two members must be the same type." 1186 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1187 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1188 // Build SPIR-V types and constant(s) if needed. 1189 MachineIRBuilder MIRBuilder(I); 1190 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1191 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1192 assert(I.getNumDefs() > 1 && "Not enought operands"); 1193 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1194 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1195 if (N > 1) 1196 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1197 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1198 Register ZeroReg = buildZerosVal(ResType, I); 1199 // A new virtual register to store the result struct. 1200 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1201 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1202 // Build the result name if needed. 1203 if (ResName.size() > 0) 1204 buildOpName(StructVReg, ResName, MIRBuilder); 1205 // Build the arithmetic with overflow instruction. 1206 MachineBasicBlock &BB = *I.getParent(); 1207 auto MIB = 1208 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1209 .addDef(StructVReg) 1210 .addUse(GR.getSPIRVTypeID(StructType)); 1211 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1212 MIB.addUse(I.getOperand(i).getReg()); 1213 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1214 // Build instructions to extract fields of the instruction's result. 1215 // A new virtual register to store the higher part of the result struct. 1216 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1217 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1218 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1219 auto MIB = 1220 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1221 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1222 .addUse(GR.getSPIRVTypeID(ResType)) 1223 .addUse(StructVReg) 1224 .addImm(i); 1225 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1226 } 1227 // Build boolean value from the higher part. 1228 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1229 .addDef(I.getOperand(1).getReg()) 1230 .addUse(BoolTypeReg) 1231 .addUse(HigherVReg) 1232 .addUse(ZeroReg) 1233 .constrainAllUses(TII, TRI, RBI); 1234 } 1235 1236 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1237 const SPIRVType *ResType, 1238 MachineInstr &I) const { 1239 bool Result = true; 1240 Register ScopeReg; 1241 Register MemSemEqReg; 1242 Register MemSemNeqReg; 1243 Register Ptr = I.getOperand(2).getReg(); 1244 if (!isa<GIntrinsic>(I)) { 1245 assert(I.hasOneMemOperand()); 1246 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1247 unsigned Scope = static_cast<uint32_t>(getMemScope( 1248 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1249 auto ScopeConstant = buildI32Constant(Scope, I); 1250 ScopeReg = ScopeConstant.first; 1251 Result &= ScopeConstant.second; 1252 1253 unsigned ScSem = static_cast<uint32_t>( 1254 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1255 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1256 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1257 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1258 MemSemEqReg = MemSemEqConstant.first; 1259 Result &= MemSemEqConstant.second; 1260 AtomicOrdering FO = MemOp->getFailureOrdering(); 1261 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1262 if (MemSemEq == MemSemNeq) 1263 MemSemNeqReg = MemSemEqReg; 1264 else { 1265 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1266 MemSemNeqReg = MemSemNeqConstant.first; 1267 Result &= MemSemNeqConstant.second; 1268 } 1269 } else { 1270 ScopeReg = I.getOperand(5).getReg(); 1271 MemSemEqReg = I.getOperand(6).getReg(); 1272 MemSemNeqReg = I.getOperand(7).getReg(); 1273 } 1274 1275 Register Cmp = I.getOperand(3).getReg(); 1276 Register Val = I.getOperand(4).getReg(); 1277 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1278 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1279 const DebugLoc &DL = I.getDebugLoc(); 1280 Result &= 1281 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1282 .addDef(ACmpRes) 1283 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1284 .addUse(Ptr) 1285 .addUse(ScopeReg) 1286 .addUse(MemSemEqReg) 1287 .addUse(MemSemNeqReg) 1288 .addUse(Val) 1289 .addUse(Cmp) 1290 .constrainAllUses(TII, TRI, RBI); 1291 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1292 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1293 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1294 .addDef(CmpSuccReg) 1295 .addUse(GR.getSPIRVTypeID(BoolTy)) 1296 .addUse(ACmpRes) 1297 .addUse(Cmp) 1298 .constrainAllUses(TII, TRI, RBI); 1299 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1300 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1301 .addDef(TmpReg) 1302 .addUse(GR.getSPIRVTypeID(ResType)) 1303 .addUse(ACmpRes) 1304 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1305 .addImm(0) 1306 .constrainAllUses(TII, TRI, RBI); 1307 return Result && 1308 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1309 .addDef(ResVReg) 1310 .addUse(GR.getSPIRVTypeID(ResType)) 1311 .addUse(CmpSuccReg) 1312 .addUse(TmpReg) 1313 .addImm(1) 1314 .constrainAllUses(TII, TRI, RBI); 1315 } 1316 1317 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1318 switch (SC) { 1319 case SPIRV::StorageClass::Workgroup: 1320 case SPIRV::StorageClass::CrossWorkgroup: 1321 case SPIRV::StorageClass::Function: 1322 return true; 1323 default: 1324 return false; 1325 } 1326 } 1327 1328 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1329 switch (SC) { 1330 case SPIRV::StorageClass::DeviceOnlyINTEL: 1331 case SPIRV::StorageClass::HostOnlyINTEL: 1332 return true; 1333 default: 1334 return false; 1335 } 1336 } 1337 1338 // Returns true ResVReg is referred only from global vars and OpName's. 1339 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1340 bool IsGRef = false; 1341 bool IsAllowedRefs = 1342 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1343 [&IsGRef](auto const &It) { 1344 unsigned Opcode = It.getOpcode(); 1345 if (Opcode == SPIRV::OpConstantComposite || 1346 Opcode == SPIRV::OpVariable || 1347 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1348 return IsGRef = true; 1349 return Opcode == SPIRV::OpName; 1350 }); 1351 return IsAllowedRefs && IsGRef; 1352 } 1353 1354 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1355 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1356 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1357 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1358 } 1359 1360 MachineInstrBuilder 1361 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1362 Register Src, Register DestType, 1363 uint32_t Opcode) const { 1364 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1365 TII.get(SPIRV::OpSpecConstantOp)) 1366 .addDef(Dest) 1367 .addUse(DestType) 1368 .addImm(Opcode) 1369 .addUse(Src); 1370 } 1371 1372 MachineInstrBuilder 1373 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1374 SPIRVType *SrcPtrTy) const { 1375 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1376 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1377 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1378 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1379 SPIRV::StorageClass::Generic), 1380 GR.getPointerSize())); 1381 MachineFunction *MF = I.getParent()->getParent(); 1382 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1383 MachineInstrBuilder MIB = buildSpecConstantOp( 1384 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1385 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1386 GR.add(MIB.getInstr(), MF, Tmp); 1387 return MIB; 1388 } 1389 1390 // In SPIR-V address space casting can only happen to and from the Generic 1391 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1392 // pointers to and from Generic pointers. As such, we can convert e.g. from 1393 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1394 // other combinations can only be done by a bitcast, and are probably not safe. 1395 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1396 const SPIRVType *ResType, 1397 MachineInstr &I) const { 1398 MachineBasicBlock &BB = *I.getParent(); 1399 const DebugLoc &DL = I.getDebugLoc(); 1400 1401 Register SrcPtr = I.getOperand(1).getReg(); 1402 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1403 1404 // don't generate a cast for a null that may be represented by OpTypeInt 1405 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1406 ResType->getOpcode() != SPIRV::OpTypePointer) 1407 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1408 .addDef(ResVReg) 1409 .addUse(SrcPtr) 1410 .constrainAllUses(TII, TRI, RBI); 1411 1412 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1413 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1414 1415 if (isASCastInGVar(MRI, ResVReg)) { 1416 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1417 // are expressed by OpSpecConstantOp with an Opcode. 1418 // TODO: maybe insert a check whether the Kernel capability was declared and 1419 // so PtrCastToGeneric/GenericCastToPtr are available. 1420 unsigned SpecOpcode = 1421 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1422 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1423 : (SrcSC == SPIRV::StorageClass::Generic && 1424 isGenericCastablePtr(DstSC) 1425 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1426 : 0); 1427 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1428 // correct value of ResType and use general i8* instead. Maybe this should 1429 // be addressed in the emit-intrinsic step to infer a correct 1430 // OpConstantComposite type. 1431 if (SpecOpcode) { 1432 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1433 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1434 .constrainAllUses(TII, TRI, RBI); 1435 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1436 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1437 return MIB.constrainAllUses(TII, TRI, RBI) && 1438 buildSpecConstantOp( 1439 I, ResVReg, MIB->getOperand(0).getReg(), 1440 getUcharPtrTypeReg(I, DstSC), 1441 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1442 .constrainAllUses(TII, TRI, RBI); 1443 } 1444 } 1445 1446 // don't generate a cast between identical storage classes 1447 if (SrcSC == DstSC) 1448 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1449 .addDef(ResVReg) 1450 .addUse(SrcPtr) 1451 .constrainAllUses(TII, TRI, RBI); 1452 1453 // Casting from an eligible pointer to Generic. 1454 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1455 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1456 // Casting from Generic to an eligible pointer. 1457 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1458 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1459 // Casting between 2 eligible pointers using Generic as an intermediary. 1460 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1461 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1462 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1463 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1464 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1465 .addDef(Tmp) 1466 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1467 .addUse(SrcPtr) 1468 .constrainAllUses(TII, TRI, RBI); 1469 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1470 .addDef(ResVReg) 1471 .addUse(GR.getSPIRVTypeID(ResType)) 1472 .addUse(Tmp) 1473 .constrainAllUses(TII, TRI, RBI); 1474 } 1475 1476 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1477 // be applied 1478 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1479 return selectUnOp(ResVReg, ResType, I, 1480 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1481 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1482 return selectUnOp(ResVReg, ResType, I, 1483 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1484 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1485 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1486 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1487 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1488 1489 // Bitcast for pointers requires that the address spaces must match 1490 return false; 1491 } 1492 1493 static unsigned getFCmpOpcode(unsigned PredNum) { 1494 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1495 switch (Pred) { 1496 case CmpInst::FCMP_OEQ: 1497 return SPIRV::OpFOrdEqual; 1498 case CmpInst::FCMP_OGE: 1499 return SPIRV::OpFOrdGreaterThanEqual; 1500 case CmpInst::FCMP_OGT: 1501 return SPIRV::OpFOrdGreaterThan; 1502 case CmpInst::FCMP_OLE: 1503 return SPIRV::OpFOrdLessThanEqual; 1504 case CmpInst::FCMP_OLT: 1505 return SPIRV::OpFOrdLessThan; 1506 case CmpInst::FCMP_ONE: 1507 return SPIRV::OpFOrdNotEqual; 1508 case CmpInst::FCMP_ORD: 1509 return SPIRV::OpOrdered; 1510 case CmpInst::FCMP_UEQ: 1511 return SPIRV::OpFUnordEqual; 1512 case CmpInst::FCMP_UGE: 1513 return SPIRV::OpFUnordGreaterThanEqual; 1514 case CmpInst::FCMP_UGT: 1515 return SPIRV::OpFUnordGreaterThan; 1516 case CmpInst::FCMP_ULE: 1517 return SPIRV::OpFUnordLessThanEqual; 1518 case CmpInst::FCMP_ULT: 1519 return SPIRV::OpFUnordLessThan; 1520 case CmpInst::FCMP_UNE: 1521 return SPIRV::OpFUnordNotEqual; 1522 case CmpInst::FCMP_UNO: 1523 return SPIRV::OpUnordered; 1524 default: 1525 llvm_unreachable("Unknown predicate type for FCmp"); 1526 } 1527 } 1528 1529 static unsigned getICmpOpcode(unsigned PredNum) { 1530 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1531 switch (Pred) { 1532 case CmpInst::ICMP_EQ: 1533 return SPIRV::OpIEqual; 1534 case CmpInst::ICMP_NE: 1535 return SPIRV::OpINotEqual; 1536 case CmpInst::ICMP_SGE: 1537 return SPIRV::OpSGreaterThanEqual; 1538 case CmpInst::ICMP_SGT: 1539 return SPIRV::OpSGreaterThan; 1540 case CmpInst::ICMP_SLE: 1541 return SPIRV::OpSLessThanEqual; 1542 case CmpInst::ICMP_SLT: 1543 return SPIRV::OpSLessThan; 1544 case CmpInst::ICMP_UGE: 1545 return SPIRV::OpUGreaterThanEqual; 1546 case CmpInst::ICMP_UGT: 1547 return SPIRV::OpUGreaterThan; 1548 case CmpInst::ICMP_ULE: 1549 return SPIRV::OpULessThanEqual; 1550 case CmpInst::ICMP_ULT: 1551 return SPIRV::OpULessThan; 1552 default: 1553 llvm_unreachable("Unknown predicate type for ICmp"); 1554 } 1555 } 1556 1557 static unsigned getPtrCmpOpcode(unsigned Pred) { 1558 switch (static_cast<CmpInst::Predicate>(Pred)) { 1559 case CmpInst::ICMP_EQ: 1560 return SPIRV::OpPtrEqual; 1561 case CmpInst::ICMP_NE: 1562 return SPIRV::OpPtrNotEqual; 1563 default: 1564 llvm_unreachable("Unknown predicate type for pointer comparison"); 1565 } 1566 } 1567 1568 // Return the logical operation, or abort if none exists. 1569 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1570 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1571 switch (Pred) { 1572 case CmpInst::ICMP_EQ: 1573 return SPIRV::OpLogicalEqual; 1574 case CmpInst::ICMP_NE: 1575 return SPIRV::OpLogicalNotEqual; 1576 default: 1577 llvm_unreachable("Unknown predicate type for Bool comparison"); 1578 } 1579 } 1580 1581 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1582 if (!LLVMFloatTy) 1583 return APFloat::getZero(APFloat::IEEEsingle()); 1584 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1585 case Type::HalfTyID: 1586 return APFloat::getZero(APFloat::IEEEhalf()); 1587 default: 1588 case Type::FloatTyID: 1589 return APFloat::getZero(APFloat::IEEEsingle()); 1590 case Type::DoubleTyID: 1591 return APFloat::getZero(APFloat::IEEEdouble()); 1592 } 1593 } 1594 1595 static APFloat getOneFP(const Type *LLVMFloatTy) { 1596 if (!LLVMFloatTy) 1597 return APFloat::getOne(APFloat::IEEEsingle()); 1598 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1599 case Type::HalfTyID: 1600 return APFloat::getOne(APFloat::IEEEhalf()); 1601 default: 1602 case Type::FloatTyID: 1603 return APFloat::getOne(APFloat::IEEEsingle()); 1604 case Type::DoubleTyID: 1605 return APFloat::getOne(APFloat::IEEEdouble()); 1606 } 1607 } 1608 1609 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1610 const SPIRVType *ResType, 1611 MachineInstr &I, 1612 unsigned OpAnyOrAll) const { 1613 assert(I.getNumOperands() == 3); 1614 assert(I.getOperand(2).isReg()); 1615 MachineBasicBlock &BB = *I.getParent(); 1616 Register InputRegister = I.getOperand(2).getReg(); 1617 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1618 1619 if (!InputType) 1620 report_fatal_error("Input Type could not be determined."); 1621 1622 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1623 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1624 if (IsBoolTy && !IsVectorTy) { 1625 assert(ResVReg == I.getOperand(0).getReg()); 1626 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1627 TII.get(TargetOpcode::COPY)) 1628 .addDef(ResVReg) 1629 .addUse(InputRegister) 1630 .constrainAllUses(TII, TRI, RBI); 1631 } 1632 1633 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1634 unsigned SpirvNotEqualId = 1635 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1636 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1637 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1638 Register NotEqualReg = ResVReg; 1639 1640 if (IsVectorTy) { 1641 NotEqualReg = IsBoolTy ? InputRegister 1642 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1643 const unsigned NumElts = InputType->getOperand(2).getImm(); 1644 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1645 } 1646 1647 bool Result = true; 1648 if (!IsBoolTy) { 1649 Register ConstZeroReg = 1650 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1651 1652 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1653 .addDef(NotEqualReg) 1654 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1655 .addUse(InputRegister) 1656 .addUse(ConstZeroReg) 1657 .constrainAllUses(TII, TRI, RBI); 1658 } 1659 1660 if (!IsVectorTy) 1661 return Result; 1662 1663 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1664 .addDef(ResVReg) 1665 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1666 .addUse(NotEqualReg) 1667 .constrainAllUses(TII, TRI, RBI); 1668 } 1669 1670 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1671 const SPIRVType *ResType, 1672 MachineInstr &I) const { 1673 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1674 } 1675 1676 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1677 const SPIRVType *ResType, 1678 MachineInstr &I) const { 1679 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1680 } 1681 1682 // Select the OpDot instruction for the given float dot 1683 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1684 const SPIRVType *ResType, 1685 MachineInstr &I) const { 1686 assert(I.getNumOperands() == 4); 1687 assert(I.getOperand(2).isReg()); 1688 assert(I.getOperand(3).isReg()); 1689 1690 [[maybe_unused]] SPIRVType *VecType = 1691 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1692 1693 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1694 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1695 "dot product requires a vector of at least 2 components"); 1696 1697 [[maybe_unused]] SPIRVType *EltType = 1698 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1699 1700 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1701 1702 MachineBasicBlock &BB = *I.getParent(); 1703 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1704 .addDef(ResVReg) 1705 .addUse(GR.getSPIRVTypeID(ResType)) 1706 .addUse(I.getOperand(2).getReg()) 1707 .addUse(I.getOperand(3).getReg()) 1708 .constrainAllUses(TII, TRI, RBI); 1709 } 1710 1711 // Since pre-1.6 SPIRV has no integer dot implementation, 1712 // expand by piecewise multiplying and adding the results 1713 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1714 const SPIRVType *ResType, 1715 MachineInstr &I) const { 1716 assert(I.getNumOperands() == 4); 1717 assert(I.getOperand(2).isReg()); 1718 assert(I.getOperand(3).isReg()); 1719 MachineBasicBlock &BB = *I.getParent(); 1720 1721 // Multiply the vectors, then sum the results 1722 Register Vec0 = I.getOperand(2).getReg(); 1723 Register Vec1 = I.getOperand(3).getReg(); 1724 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1725 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1726 1727 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1728 .addDef(TmpVec) 1729 .addUse(GR.getSPIRVTypeID(VecType)) 1730 .addUse(Vec0) 1731 .addUse(Vec1) 1732 .constrainAllUses(TII, TRI, RBI); 1733 1734 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1735 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1736 "dot product requires a vector of at least 2 components"); 1737 1738 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1739 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1740 .addDef(Res) 1741 .addUse(GR.getSPIRVTypeID(ResType)) 1742 .addUse(TmpVec) 1743 .addImm(0) 1744 .constrainAllUses(TII, TRI, RBI); 1745 1746 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1747 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1748 1749 Result &= 1750 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1751 .addDef(Elt) 1752 .addUse(GR.getSPIRVTypeID(ResType)) 1753 .addUse(TmpVec) 1754 .addImm(i) 1755 .constrainAllUses(TII, TRI, RBI); 1756 1757 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1758 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1759 : ResVReg; 1760 1761 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1762 .addDef(Sum) 1763 .addUse(GR.getSPIRVTypeID(ResType)) 1764 .addUse(Res) 1765 .addUse(Elt) 1766 .constrainAllUses(TII, TRI, RBI); 1767 Res = Sum; 1768 } 1769 1770 return Result; 1771 } 1772 1773 template <bool Signed> 1774 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1775 const SPIRVType *ResType, 1776 MachineInstr &I) const { 1777 assert(I.getNumOperands() == 5); 1778 assert(I.getOperand(2).isReg()); 1779 assert(I.getOperand(3).isReg()); 1780 assert(I.getOperand(4).isReg()); 1781 MachineBasicBlock &BB = *I.getParent(); 1782 1783 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1784 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1785 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1786 .addDef(Dot) 1787 .addUse(GR.getSPIRVTypeID(ResType)) 1788 .addUse(I.getOperand(2).getReg()) 1789 .addUse(I.getOperand(3).getReg()) 1790 .constrainAllUses(TII, TRI, RBI); 1791 1792 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1793 .addDef(ResVReg) 1794 .addUse(GR.getSPIRVTypeID(ResType)) 1795 .addUse(Dot) 1796 .addUse(I.getOperand(4).getReg()) 1797 .constrainAllUses(TII, TRI, RBI); 1798 } 1799 1800 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1801 // extract the elements of the packed inputs, multiply them and add the result 1802 // to the accumulator. 1803 template <bool Signed> 1804 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1805 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1806 assert(I.getNumOperands() == 5); 1807 assert(I.getOperand(2).isReg()); 1808 assert(I.getOperand(3).isReg()); 1809 assert(I.getOperand(4).isReg()); 1810 MachineBasicBlock &BB = *I.getParent(); 1811 1812 bool Result = true; 1813 1814 // Acc = C 1815 Register Acc = I.getOperand(4).getReg(); 1816 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1817 auto ExtractOp = 1818 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1819 1820 // Extract the i8 element, multiply and add it to the accumulator 1821 for (unsigned i = 0; i < 4; i++) { 1822 // A[i] 1823 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1824 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1825 .addDef(AElt) 1826 .addUse(GR.getSPIRVTypeID(ResType)) 1827 .addUse(I.getOperand(2).getReg()) 1828 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1829 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1830 .constrainAllUses(TII, TRI, RBI); 1831 1832 // B[i] 1833 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1834 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1835 .addDef(BElt) 1836 .addUse(GR.getSPIRVTypeID(ResType)) 1837 .addUse(I.getOperand(3).getReg()) 1838 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1839 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1840 .constrainAllUses(TII, TRI, RBI); 1841 1842 // A[i] * B[i] 1843 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1844 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1845 .addDef(Mul) 1846 .addUse(GR.getSPIRVTypeID(ResType)) 1847 .addUse(AElt) 1848 .addUse(BElt) 1849 .constrainAllUses(TII, TRI, RBI); 1850 1851 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1852 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1853 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1854 .addDef(MaskMul) 1855 .addUse(GR.getSPIRVTypeID(ResType)) 1856 .addUse(Mul) 1857 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1858 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1859 .constrainAllUses(TII, TRI, RBI); 1860 1861 // Acc = Acc + A[i] * B[i] 1862 Register Sum = 1863 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1864 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1865 .addDef(Sum) 1866 .addUse(GR.getSPIRVTypeID(ResType)) 1867 .addUse(Acc) 1868 .addUse(MaskMul) 1869 .constrainAllUses(TII, TRI, RBI); 1870 1871 Acc = Sum; 1872 } 1873 1874 return Result; 1875 } 1876 1877 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1878 /// does not have a saturate builtin. 1879 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1880 const SPIRVType *ResType, 1881 MachineInstr &I) const { 1882 assert(I.getNumOperands() == 3); 1883 assert(I.getOperand(2).isReg()); 1884 MachineBasicBlock &BB = *I.getParent(); 1885 Register VZero = buildZerosValF(ResType, I); 1886 Register VOne = buildOnesValF(ResType, I); 1887 1888 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1889 .addDef(ResVReg) 1890 .addUse(GR.getSPIRVTypeID(ResType)) 1891 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1892 .addImm(GL::FClamp) 1893 .addUse(I.getOperand(2).getReg()) 1894 .addUse(VZero) 1895 .addUse(VOne) 1896 .constrainAllUses(TII, TRI, RBI); 1897 } 1898 1899 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1900 const SPIRVType *ResType, 1901 MachineInstr &I) const { 1902 assert(I.getNumOperands() == 3); 1903 assert(I.getOperand(2).isReg()); 1904 MachineBasicBlock &BB = *I.getParent(); 1905 Register InputRegister = I.getOperand(2).getReg(); 1906 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1907 auto &DL = I.getDebugLoc(); 1908 1909 if (!InputType) 1910 report_fatal_error("Input Type could not be determined."); 1911 1912 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1913 1914 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1915 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1916 1917 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1918 1919 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1920 Register SignReg = NeedsConversion 1921 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1922 : ResVReg; 1923 1924 bool Result = 1925 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1926 .addDef(SignReg) 1927 .addUse(GR.getSPIRVTypeID(InputType)) 1928 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1929 .addImm(SignOpcode) 1930 .addUse(InputRegister) 1931 .constrainAllUses(TII, TRI, RBI); 1932 1933 if (NeedsConversion) { 1934 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1935 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1936 .addDef(ResVReg) 1937 .addUse(GR.getSPIRVTypeID(ResType)) 1938 .addUse(SignReg) 1939 .constrainAllUses(TII, TRI, RBI); 1940 } 1941 1942 return Result; 1943 } 1944 1945 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 1946 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1947 assert(I.getNumOperands() == 3); 1948 assert(I.getOperand(2).isReg()); 1949 MachineBasicBlock &BB = *I.getParent(); 1950 1951 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1952 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 1953 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 1954 1955 bool Result = 1956 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformBallot)) 1957 .addDef(BallotReg) 1958 .addUse(GR.getSPIRVTypeID(BallotType)) 1959 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1960 .addUse(I.getOperand(2).getReg()) 1961 .constrainAllUses(TII, TRI, RBI); 1962 1963 Result &= 1964 BuildMI(BB, I, I.getDebugLoc(), 1965 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 1966 .addDef(ResVReg) 1967 .addUse(GR.getSPIRVTypeID(ResType)) 1968 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1969 .addImm(SPIRV::GroupOperation::Reduce) 1970 .addUse(BallotReg) 1971 .constrainAllUses(TII, TRI, RBI); 1972 1973 return Result; 1974 } 1975 1976 bool SPIRVInstructionSelector::selectWaveReadLaneAt(Register ResVReg, 1977 const SPIRVType *ResType, 1978 MachineInstr &I) const { 1979 assert(I.getNumOperands() == 4); 1980 assert(I.getOperand(2).isReg()); 1981 assert(I.getOperand(3).isReg()); 1982 MachineBasicBlock &BB = *I.getParent(); 1983 1984 // IntTy is used to define the execution scope, set to 3 to denote a 1985 // cross-lane interaction equivalent to a SPIR-V subgroup. 1986 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1987 return BuildMI(BB, I, I.getDebugLoc(), 1988 TII.get(SPIRV::OpGroupNonUniformShuffle)) 1989 .addDef(ResVReg) 1990 .addUse(GR.getSPIRVTypeID(ResType)) 1991 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)) 1992 .addUse(I.getOperand(2).getReg()) 1993 .addUse(I.getOperand(3).getReg()); 1994 } 1995 1996 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1997 const SPIRVType *ResType, 1998 MachineInstr &I) const { 1999 MachineBasicBlock &BB = *I.getParent(); 2000 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2001 .addDef(ResVReg) 2002 .addUse(GR.getSPIRVTypeID(ResType)) 2003 .addUse(I.getOperand(1).getReg()) 2004 .constrainAllUses(TII, TRI, RBI); 2005 } 2006 2007 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2008 const SPIRVType *ResType, 2009 MachineInstr &I) const { 2010 // There is no way to implement `freeze` correctly without support on SPIR-V 2011 // standard side, but we may at least address a simple (static) case when 2012 // undef/poison value presence is obvious. The main benefit of even 2013 // incomplete `freeze` support is preventing of translation from crashing due 2014 // to lack of support on legalization and instruction selection steps. 2015 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2016 return false; 2017 Register OpReg = I.getOperand(1).getReg(); 2018 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2019 Register Reg; 2020 switch (Def->getOpcode()) { 2021 case SPIRV::ASSIGN_TYPE: 2022 if (MachineInstr *AssignToDef = 2023 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2024 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2025 Reg = Def->getOperand(2).getReg(); 2026 } 2027 break; 2028 case SPIRV::OpUndef: 2029 Reg = Def->getOperand(1).getReg(); 2030 break; 2031 } 2032 unsigned DestOpCode; 2033 if (Reg.isValid()) { 2034 DestOpCode = SPIRV::OpConstantNull; 2035 } else { 2036 DestOpCode = TargetOpcode::COPY; 2037 Reg = OpReg; 2038 } 2039 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2040 .addDef(I.getOperand(0).getReg()) 2041 .addUse(Reg) 2042 .constrainAllUses(TII, TRI, RBI); 2043 } 2044 return false; 2045 } 2046 2047 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2048 const SPIRVType *ResType) { 2049 Register OpReg = ResType->getOperand(2).getReg(); 2050 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2051 if (!OpDef) 2052 return 0; 2053 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2054 OpDef->getOperand(1).isReg()) { 2055 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2056 OpDef = RefDef; 2057 } 2058 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2059 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2060 : 0; 2061 return N; 2062 } 2063 2064 // Return true if the type represents a constant register 2065 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2066 SmallPtrSet<SPIRVType *, 4> &Visited) { 2067 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2068 OpDef->getOperand(1).isReg()) { 2069 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2070 OpDef = RefDef; 2071 } 2072 2073 if (Visited.contains(OpDef)) 2074 return true; 2075 Visited.insert(OpDef); 2076 2077 unsigned Opcode = OpDef->getOpcode(); 2078 switch (Opcode) { 2079 case TargetOpcode::G_CONSTANT: 2080 case TargetOpcode::G_FCONSTANT: 2081 return true; 2082 case TargetOpcode::G_INTRINSIC: 2083 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2084 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2085 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2086 Intrinsic::spv_const_composite; 2087 case TargetOpcode::G_BUILD_VECTOR: 2088 case TargetOpcode::G_SPLAT_VECTOR: { 2089 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2090 i++) { 2091 SPIRVType *OpNestedDef = 2092 OpDef->getOperand(i).isReg() 2093 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2094 : nullptr; 2095 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2096 return false; 2097 } 2098 return true; 2099 } 2100 } 2101 return false; 2102 } 2103 2104 // Return true if the virtual register represents a constant 2105 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2106 SmallPtrSet<SPIRVType *, 4> Visited; 2107 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2108 return isConstReg(MRI, OpDef, Visited); 2109 return false; 2110 } 2111 2112 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2113 const SPIRVType *ResType, 2114 MachineInstr &I) const { 2115 unsigned N = 0; 2116 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2117 N = GR.getScalarOrVectorComponentCount(ResType); 2118 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2119 N = getArrayComponentCount(MRI, ResType); 2120 else 2121 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2122 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2123 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2124 2125 // check if we may construct a constant vector 2126 bool IsConst = true; 2127 for (unsigned i = I.getNumExplicitDefs(); 2128 i < I.getNumExplicitOperands() && IsConst; ++i) 2129 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2130 IsConst = false; 2131 2132 if (!IsConst && N < 2) 2133 report_fatal_error( 2134 "There must be at least two constituent operands in a vector"); 2135 2136 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2137 TII.get(IsConst ? SPIRV::OpConstantComposite 2138 : SPIRV::OpCompositeConstruct)) 2139 .addDef(ResVReg) 2140 .addUse(GR.getSPIRVTypeID(ResType)); 2141 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2142 MIB.addUse(I.getOperand(i).getReg()); 2143 return MIB.constrainAllUses(TII, TRI, RBI); 2144 } 2145 2146 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2147 const SPIRVType *ResType, 2148 MachineInstr &I) const { 2149 unsigned N = 0; 2150 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2151 N = GR.getScalarOrVectorComponentCount(ResType); 2152 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2153 N = getArrayComponentCount(MRI, ResType); 2154 else 2155 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2156 2157 unsigned OpIdx = I.getNumExplicitDefs(); 2158 if (!I.getOperand(OpIdx).isReg()) 2159 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2160 2161 // check if we may construct a constant vector 2162 Register OpReg = I.getOperand(OpIdx).getReg(); 2163 bool IsConst = isConstReg(MRI, OpReg); 2164 2165 if (!IsConst && N < 2) 2166 report_fatal_error( 2167 "There must be at least two constituent operands in a vector"); 2168 2169 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2170 TII.get(IsConst ? SPIRV::OpConstantComposite 2171 : SPIRV::OpCompositeConstruct)) 2172 .addDef(ResVReg) 2173 .addUse(GR.getSPIRVTypeID(ResType)); 2174 for (unsigned i = 0; i < N; ++i) 2175 MIB.addUse(OpReg); 2176 return MIB.constrainAllUses(TII, TRI, RBI); 2177 } 2178 2179 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2180 const SPIRVType *ResType, 2181 unsigned CmpOpc, 2182 MachineInstr &I) const { 2183 Register Cmp0 = I.getOperand(2).getReg(); 2184 Register Cmp1 = I.getOperand(3).getReg(); 2185 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2186 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2187 "CMP operands should have the same type"); 2188 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2189 .addDef(ResVReg) 2190 .addUse(GR.getSPIRVTypeID(ResType)) 2191 .addUse(Cmp0) 2192 .addUse(Cmp1) 2193 .constrainAllUses(TII, TRI, RBI); 2194 } 2195 2196 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2197 const SPIRVType *ResType, 2198 MachineInstr &I) const { 2199 auto Pred = I.getOperand(1).getPredicate(); 2200 unsigned CmpOpc; 2201 2202 Register CmpOperand = I.getOperand(2).getReg(); 2203 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2204 CmpOpc = getPtrCmpOpcode(Pred); 2205 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2206 CmpOpc = getBoolCmpOpcode(Pred); 2207 else 2208 CmpOpc = getICmpOpcode(Pred); 2209 return selectCmp(ResVReg, ResType, CmpOpc, I); 2210 } 2211 2212 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2213 const MachineInstr &I, 2214 int OpIdx) const { 2215 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2216 "Expected G_FCONSTANT"); 2217 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2218 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2219 } 2220 2221 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2222 const MachineInstr &I, 2223 int OpIdx) const { 2224 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2225 "Expected G_CONSTANT"); 2226 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2227 } 2228 2229 std::pair<Register, bool> 2230 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2231 const SPIRVType *ResType) const { 2232 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2233 const SPIRVType *SpvI32Ty = 2234 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2235 // Find a constant in DT or build a new one. 2236 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2237 Register NewReg = GR.find(ConstInt, GR.CurMF); 2238 bool Result = true; 2239 if (!NewReg.isValid()) { 2240 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2241 GR.add(ConstInt, GR.CurMF, NewReg); 2242 MachineInstr *MI; 2243 MachineBasicBlock &BB = *I.getParent(); 2244 if (Val == 0) { 2245 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2246 .addDef(NewReg) 2247 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2248 } else { 2249 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2250 .addDef(NewReg) 2251 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2252 .addImm(APInt(32, Val).getZExtValue()); 2253 } 2254 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2255 } 2256 return {NewReg, Result}; 2257 } 2258 2259 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2260 const SPIRVType *ResType, 2261 MachineInstr &I) const { 2262 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2263 return selectCmp(ResVReg, ResType, CmpOp, I); 2264 } 2265 2266 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2267 MachineInstr &I) const { 2268 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2269 bool ZeroAsNull = STI.isOpenCLEnv(); 2270 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2271 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2272 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2273 } 2274 2275 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2276 MachineInstr &I) const { 2277 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2278 bool ZeroAsNull = STI.isOpenCLEnv(); 2279 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2280 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2281 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2282 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2283 } 2284 2285 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2286 MachineInstr &I) const { 2287 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2288 bool ZeroAsNull = STI.isOpenCLEnv(); 2289 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2290 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2291 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2292 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2293 } 2294 2295 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2296 const SPIRVType *ResType, 2297 MachineInstr &I) const { 2298 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2299 APInt One = 2300 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2301 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2302 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2303 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2304 } 2305 2306 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2307 const SPIRVType *ResType, 2308 MachineInstr &I, 2309 bool IsSigned) const { 2310 // To extend a bool, we need to use OpSelect between constants. 2311 Register ZeroReg = buildZerosVal(ResType, I); 2312 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2313 bool IsScalarBool = 2314 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2315 unsigned Opcode = 2316 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2317 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2318 .addDef(ResVReg) 2319 .addUse(GR.getSPIRVTypeID(ResType)) 2320 .addUse(I.getOperand(1).getReg()) 2321 .addUse(OneReg) 2322 .addUse(ZeroReg) 2323 .constrainAllUses(TII, TRI, RBI); 2324 } 2325 2326 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2327 const SPIRVType *ResType, 2328 MachineInstr &I, bool IsSigned, 2329 unsigned Opcode) const { 2330 Register SrcReg = I.getOperand(1).getReg(); 2331 // We can convert bool value directly to float type without OpConvert*ToF, 2332 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2333 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2334 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2335 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2336 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2337 const unsigned NumElts = ResType->getOperand(2).getImm(); 2338 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2339 } 2340 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2341 selectSelect(SrcReg, TmpType, I, false); 2342 } 2343 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2344 } 2345 2346 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2347 const SPIRVType *ResType, 2348 MachineInstr &I, bool IsSigned) const { 2349 Register SrcReg = I.getOperand(1).getReg(); 2350 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2351 return selectSelect(ResVReg, ResType, I, IsSigned); 2352 2353 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2354 if (SrcType == ResType) { 2355 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2356 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2357 if (DstRC != SrcRC && SrcRC) 2358 MRI->setRegClass(ResVReg, SrcRC); 2359 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2360 TII.get(TargetOpcode::COPY)) 2361 .addDef(ResVReg) 2362 .addUse(SrcReg) 2363 .constrainAllUses(TII, TRI, RBI); 2364 } 2365 2366 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2367 return selectUnOp(ResVReg, ResType, I, Opcode); 2368 } 2369 2370 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2371 Register ResVReg, 2372 MachineInstr &I, 2373 const SPIRVType *IntTy, 2374 const SPIRVType *BoolTy) const { 2375 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2376 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2377 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2378 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2379 Register Zero = buildZerosVal(IntTy, I); 2380 Register One = buildOnesVal(false, IntTy, I); 2381 MachineBasicBlock &BB = *I.getParent(); 2382 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2383 .addDef(BitIntReg) 2384 .addUse(GR.getSPIRVTypeID(IntTy)) 2385 .addUse(IntReg) 2386 .addUse(One) 2387 .constrainAllUses(TII, TRI, RBI); 2388 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2389 .addDef(ResVReg) 2390 .addUse(GR.getSPIRVTypeID(BoolTy)) 2391 .addUse(BitIntReg) 2392 .addUse(Zero) 2393 .constrainAllUses(TII, TRI, RBI); 2394 } 2395 2396 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2397 const SPIRVType *ResType, 2398 MachineInstr &I) const { 2399 Register IntReg = I.getOperand(1).getReg(); 2400 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2401 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2402 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2403 if (ArgType == ResType) { 2404 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2405 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2406 if (DstRC != SrcRC && SrcRC) 2407 MRI->setRegClass(ResVReg, SrcRC); 2408 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2409 TII.get(TargetOpcode::COPY)) 2410 .addDef(ResVReg) 2411 .addUse(IntReg) 2412 .constrainAllUses(TII, TRI, RBI); 2413 } 2414 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2415 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2416 return selectUnOp(ResVReg, ResType, I, Opcode); 2417 } 2418 2419 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2420 const SPIRVType *ResType, 2421 const APInt &Imm, 2422 MachineInstr &I) const { 2423 unsigned TyOpcode = ResType->getOpcode(); 2424 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2425 MachineBasicBlock &BB = *I.getParent(); 2426 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2427 Imm.isZero()) 2428 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2429 .addDef(ResVReg) 2430 .addUse(GR.getSPIRVTypeID(ResType)) 2431 .constrainAllUses(TII, TRI, RBI); 2432 if (TyOpcode == SPIRV::OpTypeInt) { 2433 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2434 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2435 if (Reg == ResVReg) 2436 return true; 2437 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2438 .addDef(ResVReg) 2439 .addUse(Reg) 2440 .constrainAllUses(TII, TRI, RBI); 2441 } 2442 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2443 .addDef(ResVReg) 2444 .addUse(GR.getSPIRVTypeID(ResType)); 2445 // <=32-bit integers should be caught by the sdag pattern. 2446 assert(Imm.getBitWidth() > 32); 2447 addNumImm(Imm, MIB); 2448 return MIB.constrainAllUses(TII, TRI, RBI); 2449 } 2450 2451 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2452 const SPIRVType *ResType, 2453 MachineInstr &I) const { 2454 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2455 .addDef(ResVReg) 2456 .addUse(GR.getSPIRVTypeID(ResType)) 2457 .constrainAllUses(TII, TRI, RBI); 2458 } 2459 2460 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2461 assert(MO.isReg()); 2462 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2463 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2464 assert(TypeInst->getOperand(1).isReg()); 2465 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2466 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2467 } 2468 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2469 } 2470 2471 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2472 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2473 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2474 return TypeInst->getOperand(2).getImm(); 2475 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2476 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2477 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2478 } 2479 2480 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2481 const SPIRVType *ResType, 2482 MachineInstr &I) const { 2483 MachineBasicBlock &BB = *I.getParent(); 2484 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2485 .addDef(ResVReg) 2486 .addUse(GR.getSPIRVTypeID(ResType)) 2487 // object to insert 2488 .addUse(I.getOperand(3).getReg()) 2489 // composite to insert into 2490 .addUse(I.getOperand(2).getReg()); 2491 for (unsigned i = 4; i < I.getNumOperands(); i++) 2492 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2493 return MIB.constrainAllUses(TII, TRI, RBI); 2494 } 2495 2496 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2497 const SPIRVType *ResType, 2498 MachineInstr &I) const { 2499 MachineBasicBlock &BB = *I.getParent(); 2500 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2501 .addDef(ResVReg) 2502 .addUse(GR.getSPIRVTypeID(ResType)) 2503 .addUse(I.getOperand(2).getReg()); 2504 for (unsigned i = 3; i < I.getNumOperands(); i++) 2505 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2506 return MIB.constrainAllUses(TII, TRI, RBI); 2507 } 2508 2509 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2510 const SPIRVType *ResType, 2511 MachineInstr &I) const { 2512 if (isImm(I.getOperand(4), MRI)) 2513 return selectInsertVal(ResVReg, ResType, I); 2514 MachineBasicBlock &BB = *I.getParent(); 2515 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2516 .addDef(ResVReg) 2517 .addUse(GR.getSPIRVTypeID(ResType)) 2518 .addUse(I.getOperand(2).getReg()) 2519 .addUse(I.getOperand(3).getReg()) 2520 .addUse(I.getOperand(4).getReg()) 2521 .constrainAllUses(TII, TRI, RBI); 2522 } 2523 2524 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2525 const SPIRVType *ResType, 2526 MachineInstr &I) const { 2527 if (isImm(I.getOperand(3), MRI)) 2528 return selectExtractVal(ResVReg, ResType, I); 2529 MachineBasicBlock &BB = *I.getParent(); 2530 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2531 .addDef(ResVReg) 2532 .addUse(GR.getSPIRVTypeID(ResType)) 2533 .addUse(I.getOperand(2).getReg()) 2534 .addUse(I.getOperand(3).getReg()) 2535 .constrainAllUses(TII, TRI, RBI); 2536 } 2537 2538 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2539 const SPIRVType *ResType, 2540 MachineInstr &I) const { 2541 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2542 2543 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2544 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2545 // we have to use Op[InBounds]AccessChain. 2546 const unsigned Opcode = STI.isVulkanEnv() 2547 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2548 : SPIRV::OpAccessChain) 2549 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2550 : SPIRV::OpPtrAccessChain); 2551 2552 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2553 .addDef(ResVReg) 2554 .addUse(GR.getSPIRVTypeID(ResType)) 2555 // Object to get a pointer to. 2556 .addUse(I.getOperand(3).getReg()); 2557 // Adding indices. 2558 const unsigned StartingIndex = 2559 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2560 ? 5 2561 : 4; 2562 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2563 Res.addUse(I.getOperand(i).getReg()); 2564 return Res.constrainAllUses(TII, TRI, RBI); 2565 } 2566 2567 // Maybe wrap a value into OpSpecConstantOp 2568 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2569 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2570 bool Result = true; 2571 unsigned Lim = I.getNumExplicitOperands(); 2572 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2573 Register OpReg = I.getOperand(i).getReg(); 2574 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2575 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2576 SmallPtrSet<SPIRVType *, 4> Visited; 2577 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2578 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2579 GR.isAggregateType(OpType)) { 2580 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2581 // by selectAddrSpaceCast() 2582 CompositeArgs.push_back(OpReg); 2583 continue; 2584 } 2585 MachineFunction *MF = I.getMF(); 2586 Register WrapReg = GR.find(OpDefine, MF); 2587 if (WrapReg.isValid()) { 2588 CompositeArgs.push_back(WrapReg); 2589 continue; 2590 } 2591 // Create a new register for the wrapper 2592 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2593 GR.add(OpDefine, MF, WrapReg); 2594 CompositeArgs.push_back(WrapReg); 2595 // Decorate the wrapper register and generate a new instruction 2596 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2597 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2598 MachineBasicBlock &BB = *I.getParent(); 2599 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2600 .addDef(WrapReg) 2601 .addUse(GR.getSPIRVTypeID(OpType)) 2602 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2603 .addUse(OpReg) 2604 .constrainAllUses(TII, TRI, RBI); 2605 if (!Result) 2606 break; 2607 } 2608 return Result; 2609 } 2610 2611 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2612 const SPIRVType *ResType, 2613 MachineInstr &I) const { 2614 MachineBasicBlock &BB = *I.getParent(); 2615 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2616 switch (IID) { 2617 case Intrinsic::spv_load: 2618 return selectLoad(ResVReg, ResType, I); 2619 case Intrinsic::spv_store: 2620 return selectStore(I); 2621 case Intrinsic::spv_extractv: 2622 return selectExtractVal(ResVReg, ResType, I); 2623 case Intrinsic::spv_insertv: 2624 return selectInsertVal(ResVReg, ResType, I); 2625 case Intrinsic::spv_extractelt: 2626 return selectExtractElt(ResVReg, ResType, I); 2627 case Intrinsic::spv_insertelt: 2628 return selectInsertElt(ResVReg, ResType, I); 2629 case Intrinsic::spv_gep: 2630 return selectGEP(ResVReg, ResType, I); 2631 case Intrinsic::spv_unref_global: 2632 case Intrinsic::spv_init_global: { 2633 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2634 MachineInstr *Init = I.getNumExplicitOperands() > 2 2635 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2636 : nullptr; 2637 assert(MI); 2638 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2639 } 2640 case Intrinsic::spv_undef: { 2641 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2642 .addDef(ResVReg) 2643 .addUse(GR.getSPIRVTypeID(ResType)); 2644 return MIB.constrainAllUses(TII, TRI, RBI); 2645 } 2646 case Intrinsic::spv_const_composite: { 2647 // If no values are attached, the composite is null constant. 2648 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2649 // Select a proper instruction. 2650 unsigned Opcode = SPIRV::OpConstantNull; 2651 SmallVector<Register> CompositeArgs; 2652 if (!IsNull) { 2653 Opcode = SPIRV::OpConstantComposite; 2654 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2655 return false; 2656 } 2657 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2658 .addDef(ResVReg) 2659 .addUse(GR.getSPIRVTypeID(ResType)); 2660 // skip type MD node we already used when generated assign.type for this 2661 if (!IsNull) { 2662 for (Register OpReg : CompositeArgs) 2663 MIB.addUse(OpReg); 2664 } 2665 return MIB.constrainAllUses(TII, TRI, RBI); 2666 } 2667 case Intrinsic::spv_assign_name: { 2668 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2669 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2670 for (unsigned i = I.getNumExplicitDefs() + 2; 2671 i < I.getNumExplicitOperands(); ++i) { 2672 MIB.addImm(I.getOperand(i).getImm()); 2673 } 2674 return MIB.constrainAllUses(TII, TRI, RBI); 2675 } 2676 case Intrinsic::spv_switch: { 2677 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2678 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2679 if (I.getOperand(i).isReg()) 2680 MIB.addReg(I.getOperand(i).getReg()); 2681 else if (I.getOperand(i).isCImm()) 2682 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2683 else if (I.getOperand(i).isMBB()) 2684 MIB.addMBB(I.getOperand(i).getMBB()); 2685 else 2686 llvm_unreachable("Unexpected OpSwitch operand"); 2687 } 2688 return MIB.constrainAllUses(TII, TRI, RBI); 2689 } 2690 case Intrinsic::spv_loop_merge: 2691 case Intrinsic::spv_selection_merge: { 2692 const auto Opcode = IID == Intrinsic::spv_selection_merge 2693 ? SPIRV::OpSelectionMerge 2694 : SPIRV::OpLoopMerge; 2695 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2696 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2697 assert(I.getOperand(i).isMBB()); 2698 MIB.addMBB(I.getOperand(i).getMBB()); 2699 } 2700 MIB.addImm(SPIRV::SelectionControl::None); 2701 return MIB.constrainAllUses(TII, TRI, RBI); 2702 } 2703 case Intrinsic::spv_cmpxchg: 2704 return selectAtomicCmpXchg(ResVReg, ResType, I); 2705 case Intrinsic::spv_unreachable: 2706 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2707 .constrainAllUses(TII, TRI, RBI); 2708 case Intrinsic::spv_alloca: 2709 return selectFrameIndex(ResVReg, ResType, I); 2710 case Intrinsic::spv_alloca_array: 2711 return selectAllocaArray(ResVReg, ResType, I); 2712 case Intrinsic::spv_assume: 2713 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2714 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2715 .addUse(I.getOperand(1).getReg()) 2716 .constrainAllUses(TII, TRI, RBI); 2717 break; 2718 case Intrinsic::spv_expect: 2719 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2720 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2721 .addDef(ResVReg) 2722 .addUse(GR.getSPIRVTypeID(ResType)) 2723 .addUse(I.getOperand(2).getReg()) 2724 .addUse(I.getOperand(3).getReg()) 2725 .constrainAllUses(TII, TRI, RBI); 2726 break; 2727 case Intrinsic::arithmetic_fence: 2728 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2729 return BuildMI(BB, I, I.getDebugLoc(), 2730 TII.get(SPIRV::OpArithmeticFenceEXT)) 2731 .addDef(ResVReg) 2732 .addUse(GR.getSPIRVTypeID(ResType)) 2733 .addUse(I.getOperand(2).getReg()) 2734 .constrainAllUses(TII, TRI, RBI); 2735 else 2736 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), 2737 ResVReg) 2738 .addUse(I.getOperand(2).getReg()) 2739 .constrainAllUses(TII, TRI, RBI); 2740 break; 2741 case Intrinsic::spv_thread_id: 2742 return selectSpvThreadId(ResVReg, ResType, I); 2743 case Intrinsic::spv_fdot: 2744 return selectFloatDot(ResVReg, ResType, I); 2745 case Intrinsic::spv_udot: 2746 case Intrinsic::spv_sdot: 2747 return selectIntegerDot(ResVReg, ResType, I); 2748 case Intrinsic::spv_dot4add_i8packed: 2749 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2750 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2751 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2752 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2753 case Intrinsic::spv_dot4add_u8packed: 2754 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2755 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2756 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2757 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2758 case Intrinsic::spv_all: 2759 return selectAll(ResVReg, ResType, I); 2760 case Intrinsic::spv_any: 2761 return selectAny(ResVReg, ResType, I); 2762 case Intrinsic::spv_cross: 2763 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2764 case Intrinsic::spv_lerp: 2765 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2766 case Intrinsic::spv_length: 2767 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2768 case Intrinsic::spv_degrees: 2769 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2770 case Intrinsic::spv_frac: 2771 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2772 case Intrinsic::spv_normalize: 2773 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2774 case Intrinsic::spv_rsqrt: 2775 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2776 case Intrinsic::spv_sign: 2777 return selectSign(ResVReg, ResType, I); 2778 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2779 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2780 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2781 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2782 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2783 bool Result = true; 2784 auto MemSemConstant = 2785 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2786 Register MemSemReg = MemSemConstant.first; 2787 Result &= MemSemConstant.second; 2788 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2789 Register ScopeReg = ScopeConstant.first; 2790 Result &= ScopeConstant.second; 2791 MachineBasicBlock &BB = *I.getParent(); 2792 return Result && 2793 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2794 .addUse(ScopeReg) 2795 .addUse(ScopeReg) 2796 .addUse(MemSemReg) 2797 .constrainAllUses(TII, TRI, RBI); 2798 } 2799 case Intrinsic::spv_lifetime_start: 2800 case Intrinsic::spv_lifetime_end: { 2801 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2802 : SPIRV::OpLifetimeStop; 2803 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2804 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2805 if (Size == -1) 2806 Size = 0; 2807 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2808 .addUse(PtrReg) 2809 .addImm(Size) 2810 .constrainAllUses(TII, TRI, RBI); 2811 } 2812 case Intrinsic::spv_saturate: 2813 return selectSaturate(ResVReg, ResType, I); 2814 case Intrinsic::spv_nclamp: 2815 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2816 case Intrinsic::spv_uclamp: 2817 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2818 case Intrinsic::spv_sclamp: 2819 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2820 case Intrinsic::spv_wave_active_countbits: 2821 return selectWaveActiveCountBits(ResVReg, ResType, I); 2822 case Intrinsic::spv_wave_is_first_lane: { 2823 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2824 return BuildMI(BB, I, I.getDebugLoc(), 2825 TII.get(SPIRV::OpGroupNonUniformElect)) 2826 .addDef(ResVReg) 2827 .addUse(GR.getSPIRVTypeID(ResType)) 2828 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2829 } 2830 case Intrinsic::spv_wave_readlane: 2831 return selectWaveReadLaneAt(ResVReg, ResType, I); 2832 case Intrinsic::spv_step: 2833 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2834 case Intrinsic::spv_radians: 2835 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2836 // Discard intrinsics which we do not expect to actually represent code after 2837 // lowering or intrinsics which are not implemented but should not crash when 2838 // found in a customer's LLVM IR input. 2839 case Intrinsic::instrprof_increment: 2840 case Intrinsic::instrprof_increment_step: 2841 case Intrinsic::instrprof_value_profile: 2842 break; 2843 // Discard internal intrinsics. 2844 case Intrinsic::spv_value_md: 2845 break; 2846 case Intrinsic::spv_handle_fromBinding: { 2847 return selectHandleFromBinding(ResVReg, ResType, I); 2848 } 2849 default: { 2850 std::string DiagMsg; 2851 raw_string_ostream OS(DiagMsg); 2852 I.print(OS); 2853 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2854 report_fatal_error(DiagMsg.c_str(), false); 2855 } 2856 } 2857 return true; 2858 } 2859 2860 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2861 const SPIRVType *ResType, 2862 MachineInstr &I) const { 2863 2864 uint32_t Set = foldImm(I.getOperand(2), MRI); 2865 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2866 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2867 Register IndexReg = I.getOperand(5).getReg(); 2868 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2869 2870 MachineIRBuilder MIRBuilder(I); 2871 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2872 IndexReg, IsNonUniform, MIRBuilder); 2873 2874 if (IsNonUniform) 2875 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2876 2877 // TODO: For now we assume the resource is an image, which needs to be 2878 // loaded to get the handle. That will not be true for storage buffers. 2879 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2880 .addDef(ResVReg) 2881 .addUse(GR.getSPIRVTypeID(ResType)) 2882 .addUse(VarReg) 2883 .constrainAllUses(TII, TRI, RBI); 2884 } 2885 2886 Register SPIRVInstructionSelector::buildPointerToResource( 2887 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 2888 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 2889 MachineIRBuilder MIRBuilder) const { 2890 if (ArraySize == 1) 2891 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 2892 MIRBuilder); 2893 2894 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 2895 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 2896 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 2897 VarType, Set, Binding, MIRBuilder); 2898 2899 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 2900 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 2901 2902 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2903 if (IsNonUniform) { 2904 // It is unclear which value needs to be marked an non-uniform, so both 2905 // the index and the access changed are decorated as non-uniform. 2906 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2907 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2908 } 2909 2910 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 2911 .addDef(AcReg) 2912 .addUse(GR.getSPIRVTypeID(ResPointerType)) 2913 .addUse(VarReg) 2914 .addUse(IndexReg); 2915 2916 return AcReg; 2917 } 2918 2919 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 2920 const SPIRVType *ResType, 2921 MachineInstr &I, 2922 bool IsSigned) const { 2923 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2924 // zero or sign extend 2925 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2926 bool Result = 2927 selectUnOpWithSrc(ExtReg, ResType, I, I.getOperand(2).getReg(), Opcode); 2928 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 2929 } 2930 2931 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 2932 const SPIRVType *ResType, 2933 MachineInstr &I, 2934 Register SrcReg, 2935 bool IsSigned) const { 2936 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 2937 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2938 .addDef(ResVReg) 2939 .addUse(GR.getSPIRVTypeID(ResType)) 2940 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2941 .addImm(Opcode) 2942 .addUse(SrcReg) 2943 .constrainAllUses(TII, TRI, RBI); 2944 } 2945 2946 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 2947 const SPIRVType *ResType, 2948 MachineInstr &I, 2949 bool IsSigned) const { 2950 Register OpReg = I.getOperand(2).getReg(); 2951 // 1. split our int64 into 2 pieces using a bitcast 2952 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 2953 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 2954 MachineIRBuilder MIRBuilder(I); 2955 SPIRVType *postCastT = 2956 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 2957 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 2958 bool Result = 2959 selectUnOpWithSrc(bitcastReg, postCastT, I, OpReg, SPIRV::OpBitcast); 2960 2961 // 2. call firstbithigh 2962 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 2963 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 2964 2965 // 3. split result vector into high bits and low bits 2966 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2967 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2968 2969 bool ZeroAsNull = STI.isOpenCLEnv(); 2970 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 2971 if (isScalarRes) { 2972 // if scalar do a vector extract 2973 Result &= selectNAryOpWithSrcs( 2974 HighReg, ResType, I, 2975 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 2976 SPIRV::OpVectorExtractDynamic); 2977 Result &= selectNAryOpWithSrcs( 2978 LowReg, ResType, I, 2979 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 2980 SPIRV::OpVectorExtractDynamic); 2981 } else { // vector case do a shufflevector 2982 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2983 TII.get(SPIRV::OpVectorShuffle)) 2984 .addDef(HighReg) 2985 .addUse(GR.getSPIRVTypeID(ResType)) 2986 .addUse(FBHReg) 2987 .addUse(FBHReg); 2988 // ^^ this vector will not be selected from; could be empty 2989 unsigned j; 2990 for (j = 0; j < count * 2; j += 2) { 2991 MIB.addImm(j); 2992 } 2993 Result &= MIB.constrainAllUses(TII, TRI, RBI); 2994 2995 // get low bits 2996 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2997 TII.get(SPIRV::OpVectorShuffle)) 2998 .addDef(LowReg) 2999 .addUse(GR.getSPIRVTypeID(ResType)) 3000 .addUse(FBHReg) 3001 .addUse(FBHReg); 3002 // ^^ this vector will not be selected from; could be empty 3003 for (j = 1; j < count * 2; j += 2) { 3004 MIB.addImm(j); 3005 } 3006 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3007 } 3008 3009 // 4. check if result of each top 32 bits is == -1 3010 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3011 Register NegOneReg; 3012 Register Reg0; 3013 Register Reg32; 3014 unsigned selectOp; 3015 unsigned addOp; 3016 if (isScalarRes) { 3017 NegOneReg = 3018 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3019 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3020 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3021 selectOp = SPIRV::OpSelectSISCond; 3022 addOp = SPIRV::OpIAddS; 3023 } else { 3024 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3025 NegOneReg = 3026 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3027 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3028 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3029 selectOp = SPIRV::OpSelectVIVCond; 3030 addOp = SPIRV::OpIAddV; 3031 } 3032 3033 // check if the high bits are == -1; true if -1 3034 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3035 Result &= selectNAryOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3036 SPIRV::OpIEqual); 3037 3038 // Select low bits if true in BReg, otherwise high bits 3039 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3040 Result &= selectNAryOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, 3041 selectOp); 3042 3043 // Add 32 for high bits, 0 for low bits 3044 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3045 Result &= 3046 selectNAryOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3047 3048 return Result && 3049 selectNAryOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3050 } 3051 3052 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3053 const SPIRVType *ResType, 3054 MachineInstr &I, 3055 bool IsSigned) const { 3056 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3057 Register OpReg = I.getOperand(2).getReg(); 3058 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3059 3060 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3061 case 16: 3062 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3063 case 32: 3064 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3065 case 64: 3066 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3067 default: 3068 report_fatal_error( 3069 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3070 } 3071 } 3072 3073 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3074 const SPIRVType *ResType, 3075 MachineInstr &I) const { 3076 // there was an allocation size parameter to the allocation instruction 3077 // that is not 1 3078 MachineBasicBlock &BB = *I.getParent(); 3079 return BuildMI(BB, I, I.getDebugLoc(), 3080 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3081 .addDef(ResVReg) 3082 .addUse(GR.getSPIRVTypeID(ResType)) 3083 .addUse(I.getOperand(2).getReg()) 3084 .constrainAllUses(TII, TRI, RBI); 3085 } 3086 3087 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3088 const SPIRVType *ResType, 3089 MachineInstr &I) const { 3090 // Change order of instructions if needed: all OpVariable instructions in a 3091 // function must be the first instructions in the first block 3092 MachineFunction *MF = I.getParent()->getParent(); 3093 MachineBasicBlock *MBB = &MF->front(); 3094 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 3095 bool IsHeader = false; 3096 unsigned Opcode; 3097 for (; It != E && It != I; ++It) { 3098 Opcode = It->getOpcode(); 3099 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 3100 IsHeader = true; 3101 } else if (IsHeader && 3102 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 3103 ++It; 3104 break; 3105 } 3106 } 3107 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 3108 .addDef(ResVReg) 3109 .addUse(GR.getSPIRVTypeID(ResType)) 3110 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3111 .constrainAllUses(TII, TRI, RBI); 3112 } 3113 3114 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3115 // InstructionSelector walks backwards through the instructions. We can use 3116 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3117 // first, so can generate an OpBranchConditional here. If there is no 3118 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3119 const MachineInstr *PrevI = I.getPrevNode(); 3120 MachineBasicBlock &MBB = *I.getParent(); 3121 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3122 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3123 .addUse(PrevI->getOperand(0).getReg()) 3124 .addMBB(PrevI->getOperand(1).getMBB()) 3125 .addMBB(I.getOperand(0).getMBB()) 3126 .constrainAllUses(TII, TRI, RBI); 3127 } 3128 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3129 .addMBB(I.getOperand(0).getMBB()) 3130 .constrainAllUses(TII, TRI, RBI); 3131 } 3132 3133 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3134 // InstructionSelector walks backwards through the instructions. For an 3135 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3136 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3137 // generate the OpBranchConditional in selectBranch above. 3138 // 3139 // If an OpBranchConditional has been generated, we simply return, as the work 3140 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3141 // implicit fallthrough to the next basic block, so we need to create an 3142 // OpBranchConditional with an explicit "false" argument pointing to the next 3143 // basic block that LLVM would fall through to. 3144 const MachineInstr *NextI = I.getNextNode(); 3145 // Check if this has already been successfully selected. 3146 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3147 return true; 3148 // Must be relying on implicit block fallthrough, so generate an 3149 // OpBranchConditional with the "next" basic block as the "false" target. 3150 MachineBasicBlock &MBB = *I.getParent(); 3151 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3152 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3153 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3154 .addUse(I.getOperand(0).getReg()) 3155 .addMBB(I.getOperand(1).getMBB()) 3156 .addMBB(NextMBB) 3157 .constrainAllUses(TII, TRI, RBI); 3158 } 3159 3160 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3161 const SPIRVType *ResType, 3162 MachineInstr &I) const { 3163 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3164 .addDef(ResVReg) 3165 .addUse(GR.getSPIRVTypeID(ResType)); 3166 const unsigned NumOps = I.getNumOperands(); 3167 for (unsigned i = 1; i < NumOps; i += 2) { 3168 MIB.addUse(I.getOperand(i + 0).getReg()); 3169 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3170 } 3171 return MIB.constrainAllUses(TII, TRI, RBI); 3172 } 3173 3174 bool SPIRVInstructionSelector::selectGlobalValue( 3175 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3176 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3177 MachineIRBuilder MIRBuilder(I); 3178 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3179 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3180 SPIRVType *PointerBaseType; 3181 if (GVType->isArrayTy()) { 3182 SPIRVType *ArrayElementType = 3183 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3184 SPIRV::AccessQualifier::ReadWrite, false); 3185 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3186 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3187 } else { 3188 PointerBaseType = GR.getOrCreateSPIRVType( 3189 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3190 } 3191 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3192 PointerBaseType, I, TII, 3193 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3194 3195 std::string GlobalIdent; 3196 if (!GV->hasName()) { 3197 unsigned &ID = UnnamedGlobalIDs[GV]; 3198 if (ID == 0) 3199 ID = UnnamedGlobalIDs.size(); 3200 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3201 } else { 3202 GlobalIdent = GV->getGlobalIdentifier(); 3203 } 3204 3205 // Behaviour of functions as operands depends on availability of the 3206 // corresponding extension (SPV_INTEL_function_pointers): 3207 // - If there is an extension to operate with functions as operands: 3208 // We create a proper constant operand and evaluate a correct type for a 3209 // function pointer. 3210 // - Without the required extension: 3211 // We have functions as operands in tests with blocks of instruction e.g. in 3212 // transcoding/global_block.ll. These operands are not used and should be 3213 // substituted by zero constants. Their type is expected to be always 3214 // OpTypePointer Function %uchar. 3215 if (isa<Function>(GV)) { 3216 const Constant *ConstVal = GV; 3217 MachineBasicBlock &BB = *I.getParent(); 3218 Register NewReg = GR.find(ConstVal, GR.CurMF); 3219 if (!NewReg.isValid()) { 3220 Register NewReg = ResVReg; 3221 GR.add(ConstVal, GR.CurMF, NewReg); 3222 const Function *GVFun = 3223 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3224 ? dyn_cast<Function>(GV) 3225 : nullptr; 3226 if (GVFun) { 3227 // References to a function via function pointers generate virtual 3228 // registers without a definition. We will resolve it later, during 3229 // module analysis stage. 3230 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3231 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3232 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3233 MachineInstrBuilder MB = 3234 BuildMI(BB, I, I.getDebugLoc(), 3235 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3236 .addDef(NewReg) 3237 .addUse(GR.getSPIRVTypeID(ResType)) 3238 .addUse(FuncVReg); 3239 // mapping the function pointer to the used Function 3240 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3241 return MB.constrainAllUses(TII, TRI, RBI); 3242 } 3243 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3244 .addDef(NewReg) 3245 .addUse(GR.getSPIRVTypeID(ResType)) 3246 .constrainAllUses(TII, TRI, RBI); 3247 } 3248 assert(NewReg != ResVReg); 3249 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 3250 .addDef(ResVReg) 3251 .addUse(NewReg) 3252 .constrainAllUses(TII, TRI, RBI); 3253 } 3254 auto GlobalVar = cast<GlobalVariable>(GV); 3255 assert(GlobalVar->getName() != "llvm.global.annotations"); 3256 3257 bool HasInit = GlobalVar->hasInitializer() && 3258 !isa<UndefValue>(GlobalVar->getInitializer()); 3259 // Skip empty declaration for GVs with initilaizers till we get the decl with 3260 // passed initializer. 3261 if (HasInit && !Init) 3262 return true; 3263 3264 unsigned AddrSpace = GV->getAddressSpace(); 3265 SPIRV::StorageClass::StorageClass Storage = 3266 addressSpaceToStorageClass(AddrSpace, STI); 3267 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 3268 Storage != SPIRV::StorageClass::Function; 3269 SPIRV::LinkageType::LinkageType LnkType = 3270 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3271 ? SPIRV::LinkageType::Import 3272 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3273 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3274 ? SPIRV::LinkageType::LinkOnceODR 3275 : SPIRV::LinkageType::Export); 3276 3277 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 3278 Storage, Init, GlobalVar->isConstant(), 3279 HasLnkTy, LnkType, MIRBuilder, true); 3280 return Reg.isValid(); 3281 } 3282 3283 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3284 const SPIRVType *ResType, 3285 MachineInstr &I) const { 3286 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3287 return selectExtInst(ResVReg, ResType, I, CL::log10); 3288 } 3289 3290 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3291 // is implemented as: 3292 // log10(x) = log2(x) * (1 / log2(10)) 3293 // = log2(x) * 0.30103 3294 3295 MachineIRBuilder MIRBuilder(I); 3296 MachineBasicBlock &BB = *I.getParent(); 3297 3298 // Build log2(x). 3299 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3300 bool Result = 3301 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3302 .addDef(VarReg) 3303 .addUse(GR.getSPIRVTypeID(ResType)) 3304 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3305 .addImm(GL::Log2) 3306 .add(I.getOperand(1)) 3307 .constrainAllUses(TII, TRI, RBI); 3308 3309 // Build 0.30103. 3310 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3311 ResType->getOpcode() == SPIRV::OpTypeFloat); 3312 // TODO: Add matrix implementation once supported by the HLSL frontend. 3313 const SPIRVType *SpirvScalarType = 3314 ResType->getOpcode() == SPIRV::OpTypeVector 3315 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3316 : ResType; 3317 Register ScaleReg = 3318 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3319 3320 // Multiply log2(x) by 0.30103 to get log10(x) result. 3321 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3322 ? SPIRV::OpVectorTimesScalar 3323 : SPIRV::OpFMulS; 3324 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3325 .addDef(ResVReg) 3326 .addUse(GR.getSPIRVTypeID(ResType)) 3327 .addUse(VarReg) 3328 .addUse(ScaleReg) 3329 .constrainAllUses(TII, TRI, RBI); 3330 } 3331 3332 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3333 const SPIRVType *ResType, 3334 MachineInstr &I) const { 3335 // DX intrinsic: @llvm.dx.thread.id(i32) 3336 // ID Name Description 3337 // 93 ThreadId reads the thread ID 3338 3339 MachineIRBuilder MIRBuilder(I); 3340 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3341 const SPIRVType *Vec3Ty = 3342 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3343 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3344 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3345 3346 // Create new register for GlobalInvocationID builtin variable. 3347 Register NewRegister = 3348 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3349 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3350 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3351 3352 // Build GlobalInvocationID global variable with the necessary decorations. 3353 Register Variable = GR.buildGlobalVariable( 3354 NewRegister, PtrType, 3355 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3356 SPIRV::StorageClass::Input, nullptr, true, true, 3357 SPIRV::LinkageType::Import, MIRBuilder, false); 3358 3359 // Create new register for loading value. 3360 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3361 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3362 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3363 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3364 3365 // Load v3uint value from the global variable. 3366 bool Result = 3367 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3368 .addDef(LoadedRegister) 3369 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3370 .addUse(Variable); 3371 3372 // Get Thread ID index. Expecting operand is a constant immediate value, 3373 // wrapped in a type assignment. 3374 assert(I.getOperand(2).isReg()); 3375 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3376 3377 // Extract the thread ID from the loaded vector value. 3378 MachineBasicBlock &BB = *I.getParent(); 3379 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3380 .addDef(ResVReg) 3381 .addUse(GR.getSPIRVTypeID(ResType)) 3382 .addUse(LoadedRegister) 3383 .addImm(ThreadId); 3384 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3385 } 3386 3387 namespace llvm { 3388 InstructionSelector * 3389 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3390 const SPIRVSubtarget &Subtarget, 3391 const RegisterBankInfo &RBI) { 3392 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3393 } 3394 } // namespace llvm 3395