1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/Register.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/IR/IntrinsicsSPIRV.h" 34 #include "llvm/Support/Debug.h" 35 36 #define DEBUG_TYPE "spirv-isel" 37 38 using namespace llvm; 39 namespace CL = SPIRV::OpenCLExtInst; 40 namespace GL = SPIRV::GLSLExtInst; 41 42 using ExtInstList = 43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 44 45 namespace { 46 47 #define GET_GLOBALISEL_PREDICATE_BITSET 48 #include "SPIRVGenGlobalISel.inc" 49 #undef GET_GLOBALISEL_PREDICATE_BITSET 50 51 class SPIRVInstructionSelector : public InstructionSelector { 52 const SPIRVSubtarget &STI; 53 const SPIRVInstrInfo &TII; 54 const SPIRVRegisterInfo &TRI; 55 const RegisterBankInfo &RBI; 56 SPIRVGlobalRegistry &GR; 57 MachineRegisterInfo *MRI; 58 MachineFunction *HasVRegsReset = nullptr; 59 60 /// We need to keep track of the number we give to anonymous global values to 61 /// generate the same name every time when this is needed. 62 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 63 64 public: 65 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 66 const SPIRVSubtarget &ST, 67 const RegisterBankInfo &RBI); 68 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 69 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 70 BlockFrequencyInfo *BFI) override; 71 // Common selection code. Instruction-specific selection occurs in spvSelect. 72 bool select(MachineInstr &I) override; 73 static const char *getName() { return DEBUG_TYPE; } 74 75 #define GET_GLOBALISEL_PREDICATES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_PREDICATES_DECL 78 79 #define GET_GLOBALISEL_TEMPORARIES_DECL 80 #include "SPIRVGenGlobalISel.inc" 81 #undef GET_GLOBALISEL_TEMPORARIES_DECL 82 83 private: 84 void resetVRegsType(MachineFunction &MF); 85 86 // tblgen-erated 'select' implementation, used as the initial selector for 87 // the patterns that don't require complex C++. 88 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 89 90 // All instruction-specific selection that didn't happen in "select()". 91 // Is basically a large Switch/Case delegating to all other select method. 92 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I) const; 94 95 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 96 const MachineInstr *Init = nullptr) const; 97 98 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I, Register SrcReg, 100 unsigned Opcode) const; 101 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 102 unsigned Opcode) const; 103 104 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 105 MachineInstr &I) const; 106 107 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 108 MachineInstr &I) const; 109 bool selectStore(MachineInstr &I) const; 110 111 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I) const; 113 bool selectStackRestore(MachineInstr &I) const; 114 115 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 116 117 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 118 MachineInstr &I, unsigned NewOpcode, 119 unsigned NegateOpcode = 0) const; 120 121 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 bool selectFence(MachineInstr &I) const; 125 126 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 129 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 130 MachineInstr &I, unsigned OpType) const; 131 132 bool selectAll(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I) const; 134 135 bool selectAny(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I) const; 137 138 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 139 MachineInstr &I) const; 140 141 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 142 MachineInstr &I) const; 143 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 144 MachineInstr &I) const; 145 146 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 147 unsigned comparisonOpcode, MachineInstr &I) const; 148 bool selectCross(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectSign(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 158 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 162 MachineInstr &I, unsigned Opcode) const; 163 164 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 168 int OpIdx) const; 169 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 170 int OpIdx) const; 171 172 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 173 MachineInstr &I) const; 174 175 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 176 bool IsSigned) const; 177 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 178 bool IsSigned, unsigned Opcode) const; 179 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 180 bool IsSigned) const; 181 182 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I) const; 184 185 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 186 const SPIRVType *intTy, const SPIRVType *boolTy) const; 187 188 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 191 MachineInstr &I) const; 192 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 195 MachineInstr &I) const; 196 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 199 MachineInstr &I) const; 200 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 201 MachineInstr &I) const; 202 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I) const; 204 205 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I) const; 207 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 208 MachineInstr &I) const; 209 210 bool selectBranch(MachineInstr &I) const; 211 bool selectBranchCond(MachineInstr &I) const; 212 213 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 214 MachineInstr &I) const; 215 216 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 218 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I, CL::OpenCLExtInst CLInst, 220 GL::GLSLExtInst GLInst) const; 221 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I, const ExtInstList &ExtInsts) const; 223 224 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 227 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I) const; 229 230 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 233 bool selectUnmergeValues(MachineInstr &I) const; 234 235 void selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 236 MachineInstr &I) const; 237 238 // Utilities 239 Register buildI32Constant(uint32_t Val, MachineInstr &I, 240 const SPIRVType *ResType = nullptr) const; 241 242 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 243 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 244 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 245 MachineInstr &I) const; 246 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 247 248 bool wrapIntoSpecConstantOp(MachineInstr &I, 249 SmallVector<Register> &CompositeArgs) const; 250 251 Register getUcharPtrTypeReg(MachineInstr &I, 252 SPIRV::StorageClass::StorageClass SC) const; 253 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 254 Register Src, Register DestType, 255 uint32_t Opcode) const; 256 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 257 SPIRVType *SrcPtrTy) const; 258 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 259 uint32_t Binding, uint32_t ArraySize, 260 MachineIRBuilder MIRBuilder) const; 261 }; 262 263 } // end anonymous namespace 264 265 #define GET_GLOBALISEL_IMPL 266 #include "SPIRVGenGlobalISel.inc" 267 #undef GET_GLOBALISEL_IMPL 268 269 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 270 const SPIRVSubtarget &ST, 271 const RegisterBankInfo &RBI) 272 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 273 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 274 #define GET_GLOBALISEL_PREDICATES_INIT 275 #include "SPIRVGenGlobalISel.inc" 276 #undef GET_GLOBALISEL_PREDICATES_INIT 277 #define GET_GLOBALISEL_TEMPORARIES_INIT 278 #include "SPIRVGenGlobalISel.inc" 279 #undef GET_GLOBALISEL_TEMPORARIES_INIT 280 { 281 } 282 283 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 284 CodeGenCoverage *CoverageInfo, 285 ProfileSummaryInfo *PSI, 286 BlockFrequencyInfo *BFI) { 287 MRI = &MF.getRegInfo(); 288 GR.setCurrentFunc(MF); 289 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 290 } 291 292 // Ensure that register classes correspond to pattern matching rules. 293 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 294 if (HasVRegsReset == &MF) 295 return; 296 HasVRegsReset = &MF; 297 298 MachineRegisterInfo &MRI = MF.getRegInfo(); 299 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 300 Register Reg = Register::index2VirtReg(I); 301 LLT RegType = MRI.getType(Reg); 302 if (RegType.isScalar()) 303 MRI.setType(Reg, LLT::scalar(64)); 304 else if (RegType.isPointer()) 305 MRI.setType(Reg, LLT::pointer(0, 64)); 306 else if (RegType.isVector()) 307 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 308 } 309 for (const auto &MBB : MF) { 310 for (const auto &MI : MBB) { 311 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 312 continue; 313 Register DstReg = MI.getOperand(0).getReg(); 314 LLT DstType = MRI.getType(DstReg); 315 Register SrcReg = MI.getOperand(1).getReg(); 316 LLT SrcType = MRI.getType(SrcReg); 317 if (DstType != SrcType) 318 MRI.setType(DstReg, MRI.getType(SrcReg)); 319 320 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 321 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 322 if (DstRC != SrcRC && SrcRC) 323 MRI.setRegClass(DstReg, SrcRC); 324 } 325 } 326 } 327 328 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 329 330 // Defined in SPIRVLegalizerInfo.cpp. 331 extern bool isTypeFoldingSupported(unsigned Opcode); 332 333 bool SPIRVInstructionSelector::select(MachineInstr &I) { 334 resetVRegsType(*I.getParent()->getParent()); 335 336 assert(I.getParent() && "Instruction should be in a basic block!"); 337 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 338 339 Register Opcode = I.getOpcode(); 340 // If it's not a GMIR instruction, we've selected it already. 341 if (!isPreISelGenericOpcode(Opcode)) { 342 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 343 Register DstReg = I.getOperand(0).getReg(); 344 Register SrcReg = I.getOperand(1).getReg(); 345 auto *Def = MRI->getVRegDef(SrcReg); 346 if (isTypeFoldingSupported(Def->getOpcode())) { 347 bool Res = selectImpl(I, *CoverageInfo); 348 LLVM_DEBUG({ 349 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 350 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 351 I.print(dbgs()); 352 } 353 }); 354 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 355 if (Res) 356 return Res; 357 } 358 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 359 MRI->replaceRegWith(SrcReg, DstReg); 360 I.removeFromParent(); 361 return true; 362 } else if (I.getNumDefs() == 1) { 363 // Make all vregs 64 bits (for SPIR-V IDs). 364 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 365 } 366 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 367 } 368 369 if (I.getNumOperands() != I.getNumExplicitOperands()) { 370 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 371 return false; 372 } 373 374 // Common code for getting return reg+type, and removing selected instr 375 // from parent occurs here. Instr-specific selection happens in spvSelect(). 376 bool HasDefs = I.getNumDefs() > 0; 377 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 378 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 379 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 380 if (spvSelect(ResVReg, ResType, I)) { 381 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 382 for (unsigned i = 0; i < I.getNumDefs(); ++i) 383 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 384 I.removeFromParent(); 385 return true; 386 } 387 return false; 388 } 389 390 static bool mayApplyGenericSelection(unsigned Opcode) { 391 switch (Opcode) { 392 case TargetOpcode::G_CONSTANT: 393 return false; 394 case TargetOpcode::G_SADDO: 395 case TargetOpcode::G_SSUBO: 396 return true; 397 } 398 return isTypeFoldingSupported(Opcode); 399 } 400 401 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 402 const SPIRVType *ResType, 403 MachineInstr &I) const { 404 const unsigned Opcode = I.getOpcode(); 405 if (mayApplyGenericSelection(Opcode)) 406 return selectImpl(I, *CoverageInfo); 407 switch (Opcode) { 408 case TargetOpcode::G_CONSTANT: 409 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 410 I); 411 case TargetOpcode::G_GLOBAL_VALUE: 412 return selectGlobalValue(ResVReg, I); 413 case TargetOpcode::G_IMPLICIT_DEF: 414 return selectOpUndef(ResVReg, ResType, I); 415 case TargetOpcode::G_FREEZE: 416 return selectFreeze(ResVReg, ResType, I); 417 418 case TargetOpcode::G_INTRINSIC: 419 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 420 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 421 return selectIntrinsic(ResVReg, ResType, I); 422 case TargetOpcode::G_BITREVERSE: 423 return selectBitreverse(ResVReg, ResType, I); 424 425 case TargetOpcode::G_BUILD_VECTOR: 426 return selectBuildVector(ResVReg, ResType, I); 427 case TargetOpcode::G_SPLAT_VECTOR: 428 return selectSplatVector(ResVReg, ResType, I); 429 430 case TargetOpcode::G_SHUFFLE_VECTOR: { 431 MachineBasicBlock &BB = *I.getParent(); 432 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 433 .addDef(ResVReg) 434 .addUse(GR.getSPIRVTypeID(ResType)) 435 .addUse(I.getOperand(1).getReg()) 436 .addUse(I.getOperand(2).getReg()); 437 for (auto V : I.getOperand(3).getShuffleMask()) 438 MIB.addImm(V); 439 return MIB.constrainAllUses(TII, TRI, RBI); 440 } 441 case TargetOpcode::G_MEMMOVE: 442 case TargetOpcode::G_MEMCPY: 443 case TargetOpcode::G_MEMSET: 444 return selectMemOperation(ResVReg, I); 445 446 case TargetOpcode::G_ICMP: 447 return selectICmp(ResVReg, ResType, I); 448 case TargetOpcode::G_FCMP: 449 return selectFCmp(ResVReg, ResType, I); 450 451 case TargetOpcode::G_FRAME_INDEX: 452 return selectFrameIndex(ResVReg, ResType, I); 453 454 case TargetOpcode::G_LOAD: 455 return selectLoad(ResVReg, ResType, I); 456 case TargetOpcode::G_STORE: 457 return selectStore(I); 458 459 case TargetOpcode::G_BR: 460 return selectBranch(I); 461 case TargetOpcode::G_BRCOND: 462 return selectBranchCond(I); 463 464 case TargetOpcode::G_PHI: 465 return selectPhi(ResVReg, ResType, I); 466 467 case TargetOpcode::G_FPTOSI: 468 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 469 case TargetOpcode::G_FPTOUI: 470 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 471 472 case TargetOpcode::G_SITOFP: 473 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 474 case TargetOpcode::G_UITOFP: 475 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 476 477 case TargetOpcode::G_CTPOP: 478 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 479 case TargetOpcode::G_SMIN: 480 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 481 case TargetOpcode::G_UMIN: 482 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 483 484 case TargetOpcode::G_SMAX: 485 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 486 case TargetOpcode::G_UMAX: 487 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 488 489 case TargetOpcode::G_FMA: 490 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 491 492 case TargetOpcode::G_FPOW: 493 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 494 case TargetOpcode::G_FPOWI: 495 return selectExtInst(ResVReg, ResType, I, CL::pown); 496 497 case TargetOpcode::G_FEXP: 498 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 499 case TargetOpcode::G_FEXP2: 500 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 501 502 case TargetOpcode::G_FLOG: 503 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 504 case TargetOpcode::G_FLOG2: 505 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 506 case TargetOpcode::G_FLOG10: 507 return selectLog10(ResVReg, ResType, I); 508 509 case TargetOpcode::G_FABS: 510 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 511 case TargetOpcode::G_ABS: 512 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 513 514 case TargetOpcode::G_FMINNUM: 515 case TargetOpcode::G_FMINIMUM: 516 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 517 case TargetOpcode::G_FMAXNUM: 518 case TargetOpcode::G_FMAXIMUM: 519 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 520 521 case TargetOpcode::G_FCOPYSIGN: 522 return selectExtInst(ResVReg, ResType, I, CL::copysign); 523 524 case TargetOpcode::G_FCEIL: 525 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 526 case TargetOpcode::G_FFLOOR: 527 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 528 529 case TargetOpcode::G_FCOS: 530 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 531 case TargetOpcode::G_FSIN: 532 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 533 case TargetOpcode::G_FTAN: 534 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 535 case TargetOpcode::G_FACOS: 536 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 537 case TargetOpcode::G_FASIN: 538 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 539 case TargetOpcode::G_FATAN: 540 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 541 case TargetOpcode::G_FATAN2: 542 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 543 case TargetOpcode::G_FCOSH: 544 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 545 case TargetOpcode::G_FSINH: 546 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 547 case TargetOpcode::G_FTANH: 548 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 549 550 case TargetOpcode::G_FSQRT: 551 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 552 553 case TargetOpcode::G_CTTZ: 554 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 555 return selectExtInst(ResVReg, ResType, I, CL::ctz); 556 case TargetOpcode::G_CTLZ: 557 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 558 return selectExtInst(ResVReg, ResType, I, CL::clz); 559 560 case TargetOpcode::G_INTRINSIC_ROUND: 561 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 562 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 563 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 564 case TargetOpcode::G_INTRINSIC_TRUNC: 565 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 566 case TargetOpcode::G_FRINT: 567 case TargetOpcode::G_FNEARBYINT: 568 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 569 570 case TargetOpcode::G_SMULH: 571 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 572 case TargetOpcode::G_UMULH: 573 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 574 575 case TargetOpcode::G_SADDSAT: 576 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 577 case TargetOpcode::G_UADDSAT: 578 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 579 case TargetOpcode::G_SSUBSAT: 580 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 581 case TargetOpcode::G_USUBSAT: 582 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 583 584 case TargetOpcode::G_UADDO: 585 return selectOverflowArith(ResVReg, ResType, I, 586 ResType->getOpcode() == SPIRV::OpTypeVector 587 ? SPIRV::OpIAddCarryV 588 : SPIRV::OpIAddCarryS); 589 case TargetOpcode::G_USUBO: 590 return selectOverflowArith(ResVReg, ResType, I, 591 ResType->getOpcode() == SPIRV::OpTypeVector 592 ? SPIRV::OpISubBorrowV 593 : SPIRV::OpISubBorrowS); 594 case TargetOpcode::G_UMULO: 595 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 596 case TargetOpcode::G_SMULO: 597 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 598 599 case TargetOpcode::G_SEXT: 600 return selectExt(ResVReg, ResType, I, true); 601 case TargetOpcode::G_ANYEXT: 602 case TargetOpcode::G_ZEXT: 603 return selectExt(ResVReg, ResType, I, false); 604 case TargetOpcode::G_TRUNC: 605 return selectTrunc(ResVReg, ResType, I); 606 case TargetOpcode::G_FPTRUNC: 607 case TargetOpcode::G_FPEXT: 608 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 609 610 case TargetOpcode::G_PTRTOINT: 611 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 612 case TargetOpcode::G_INTTOPTR: 613 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 614 case TargetOpcode::G_BITCAST: 615 return selectBitcast(ResVReg, ResType, I); 616 case TargetOpcode::G_ADDRSPACE_CAST: 617 return selectAddrSpaceCast(ResVReg, ResType, I); 618 case TargetOpcode::G_PTR_ADD: { 619 // Currently, we get G_PTR_ADD only applied to global variables. 620 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 621 Register GV = I.getOperand(1).getReg(); 622 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 623 (void)II; 624 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 625 (*II).getOpcode() == TargetOpcode::COPY || 626 (*II).getOpcode() == SPIRV::OpVariable) && 627 isImm(I.getOperand(2), MRI)); 628 // It may be the initialization of a global variable. 629 bool IsGVInit = false; 630 for (MachineRegisterInfo::use_instr_iterator 631 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 632 UseEnd = MRI->use_instr_end(); 633 UseIt != UseEnd; UseIt = std::next(UseIt)) { 634 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 635 (*UseIt).getOpcode() == SPIRV::OpVariable) { 636 IsGVInit = true; 637 break; 638 } 639 } 640 MachineBasicBlock &BB = *I.getParent(); 641 if (!IsGVInit) { 642 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 643 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 644 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 645 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 646 // Build a new virtual register that is associated with the required 647 // data type. 648 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 649 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 650 // Having a correctly typed base we are ready to build the actually 651 // required GEP. It may not be a constant though, because all Operands 652 // of OpSpecConstantOp is to originate from other const instructions, 653 // and only the AccessChain named opcodes accept a global OpVariable 654 // instruction. We can't use an AccessChain opcode because of the type 655 // mismatch between result and base types. 656 if (!GR.isBitcastCompatible(ResType, GVType)) 657 report_fatal_error( 658 "incompatible result and operand types in a bitcast"); 659 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 660 MachineInstrBuilder MIB = 661 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 662 .addDef(NewVReg) 663 .addUse(ResTypeReg) 664 .addUse(GV); 665 return MIB.constrainAllUses(TII, TRI, RBI) && 666 BuildMI(BB, I, I.getDebugLoc(), 667 TII.get(STI.isVulkanEnv() 668 ? SPIRV::OpInBoundsAccessChain 669 : SPIRV::OpInBoundsPtrAccessChain)) 670 .addDef(ResVReg) 671 .addUse(ResTypeReg) 672 .addUse(NewVReg) 673 .addUse(I.getOperand(2).getReg()) 674 .constrainAllUses(TII, TRI, RBI); 675 } else { 676 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 677 .addDef(ResVReg) 678 .addUse(GR.getSPIRVTypeID(ResType)) 679 .addImm( 680 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 681 .addUse(GV) 682 .addUse(I.getOperand(2).getReg()) 683 .constrainAllUses(TII, TRI, RBI); 684 } 685 } 686 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 687 // initialize a global variable with a constant expression (e.g., the test 688 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 689 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 690 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 691 .addDef(ResVReg) 692 .addUse(GR.getSPIRVTypeID(ResType)) 693 .addImm(static_cast<uint32_t>( 694 SPIRV::Opcode::InBoundsPtrAccessChain)) 695 .addUse(GV) 696 .addUse(Idx) 697 .addUse(I.getOperand(2).getReg()); 698 return MIB.constrainAllUses(TII, TRI, RBI); 699 } 700 701 case TargetOpcode::G_ATOMICRMW_OR: 702 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 703 case TargetOpcode::G_ATOMICRMW_ADD: 704 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 705 case TargetOpcode::G_ATOMICRMW_AND: 706 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 707 case TargetOpcode::G_ATOMICRMW_MAX: 708 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 709 case TargetOpcode::G_ATOMICRMW_MIN: 710 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 711 case TargetOpcode::G_ATOMICRMW_SUB: 712 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 713 case TargetOpcode::G_ATOMICRMW_XOR: 714 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 715 case TargetOpcode::G_ATOMICRMW_UMAX: 716 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 717 case TargetOpcode::G_ATOMICRMW_UMIN: 718 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 719 case TargetOpcode::G_ATOMICRMW_XCHG: 720 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 721 case TargetOpcode::G_ATOMIC_CMPXCHG: 722 return selectAtomicCmpXchg(ResVReg, ResType, I); 723 724 case TargetOpcode::G_ATOMICRMW_FADD: 725 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 726 case TargetOpcode::G_ATOMICRMW_FSUB: 727 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 728 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 729 SPIRV::OpFNegate); 730 case TargetOpcode::G_ATOMICRMW_FMIN: 731 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 732 case TargetOpcode::G_ATOMICRMW_FMAX: 733 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 734 735 case TargetOpcode::G_FENCE: 736 return selectFence(I); 737 738 case TargetOpcode::G_STACKSAVE: 739 return selectStackSave(ResVReg, ResType, I); 740 case TargetOpcode::G_STACKRESTORE: 741 return selectStackRestore(I); 742 743 case TargetOpcode::G_UNMERGE_VALUES: 744 return selectUnmergeValues(I); 745 746 // Discard gen opcodes for intrinsics which we do not expect to actually 747 // represent code after lowering or intrinsics which are not implemented but 748 // should not crash when found in a customer's LLVM IR input. 749 case TargetOpcode::G_TRAP: 750 case TargetOpcode::G_DEBUGTRAP: 751 case TargetOpcode::G_UBSANTRAP: 752 case TargetOpcode::DBG_LABEL: 753 return true; 754 755 default: 756 return false; 757 } 758 } 759 760 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 761 const SPIRVType *ResType, 762 MachineInstr &I, 763 CL::OpenCLExtInst CLInst) const { 764 return selectExtInst(ResVReg, ResType, I, 765 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 766 } 767 768 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 769 const SPIRVType *ResType, 770 MachineInstr &I, 771 CL::OpenCLExtInst CLInst, 772 GL::GLSLExtInst GLInst) const { 773 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 774 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 775 return selectExtInst(ResVReg, ResType, I, ExtInsts); 776 } 777 778 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 779 const SPIRVType *ResType, 780 MachineInstr &I, 781 const ExtInstList &Insts) const { 782 783 for (const auto &Ex : Insts) { 784 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 785 uint32_t Opcode = Ex.second; 786 if (STI.canUseExtInstSet(Set)) { 787 MachineBasicBlock &BB = *I.getParent(); 788 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 789 .addDef(ResVReg) 790 .addUse(GR.getSPIRVTypeID(ResType)) 791 .addImm(static_cast<uint32_t>(Set)) 792 .addImm(Opcode); 793 const unsigned NumOps = I.getNumOperands(); 794 unsigned Index = 1; 795 if (Index < NumOps && 796 I.getOperand(Index).getType() == 797 MachineOperand::MachineOperandType::MO_IntrinsicID) 798 Index = 2; 799 for (; Index < NumOps; ++Index) 800 MIB.add(I.getOperand(Index)); 801 return MIB.constrainAllUses(TII, TRI, RBI); 802 } 803 } 804 return false; 805 } 806 807 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 808 const SPIRVType *ResType, 809 MachineInstr &I, 810 Register SrcReg, 811 unsigned Opcode) const { 812 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 813 .addDef(ResVReg) 814 .addUse(GR.getSPIRVTypeID(ResType)) 815 .addUse(SrcReg) 816 .constrainAllUses(TII, TRI, RBI); 817 } 818 819 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 820 const SPIRVType *ResType, 821 MachineInstr &I, 822 unsigned Opcode) const { 823 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 824 Register SrcReg = I.getOperand(1).getReg(); 825 bool IsGV = false; 826 for (MachineRegisterInfo::def_instr_iterator DefIt = 827 MRI->def_instr_begin(SrcReg); 828 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 829 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 830 IsGV = true; 831 break; 832 } 833 } 834 if (IsGV) { 835 uint32_t SpecOpcode = 0; 836 switch (Opcode) { 837 case SPIRV::OpConvertPtrToU: 838 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 839 break; 840 case SPIRV::OpConvertUToPtr: 841 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 842 break; 843 } 844 if (SpecOpcode) 845 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 846 TII.get(SPIRV::OpSpecConstantOp)) 847 .addDef(ResVReg) 848 .addUse(GR.getSPIRVTypeID(ResType)) 849 .addImm(SpecOpcode) 850 .addUse(SrcReg) 851 .constrainAllUses(TII, TRI, RBI); 852 } 853 } 854 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 855 Opcode); 856 } 857 858 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 859 const SPIRVType *ResType, 860 MachineInstr &I) const { 861 Register OpReg = I.getOperand(1).getReg(); 862 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 863 if (!GR.isBitcastCompatible(ResType, OpType)) 864 report_fatal_error("incompatible result and operand types in a bitcast"); 865 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 866 } 867 868 static void addMemoryOperands(MachineMemOperand *MemOp, 869 MachineInstrBuilder &MIB) { 870 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 871 if (MemOp->isVolatile()) 872 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 873 if (MemOp->isNonTemporal()) 874 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 875 if (MemOp->getAlign().value()) 876 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 877 878 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 879 MIB.addImm(SpvMemOp); 880 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 881 MIB.addImm(MemOp->getAlign().value()); 882 } 883 } 884 885 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 886 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 887 if (Flags & MachineMemOperand::Flags::MOVolatile) 888 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 889 if (Flags & MachineMemOperand::Flags::MONonTemporal) 890 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 891 892 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 893 MIB.addImm(SpvMemOp); 894 } 895 896 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 897 const SPIRVType *ResType, 898 MachineInstr &I) const { 899 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 900 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 901 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 902 .addDef(ResVReg) 903 .addUse(GR.getSPIRVTypeID(ResType)) 904 .addUse(Ptr); 905 if (!I.getNumMemOperands()) { 906 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 907 I.getOpcode() == 908 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 909 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 910 } else { 911 addMemoryOperands(*I.memoperands_begin(), MIB); 912 } 913 return MIB.constrainAllUses(TII, TRI, RBI); 914 } 915 916 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 917 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 918 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 919 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 920 MachineBasicBlock &BB = *I.getParent(); 921 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 922 .addUse(Ptr) 923 .addUse(StoreVal); 924 if (!I.getNumMemOperands()) { 925 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 926 I.getOpcode() == 927 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 928 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 929 } else { 930 addMemoryOperands(*I.memoperands_begin(), MIB); 931 } 932 return MIB.constrainAllUses(TII, TRI, RBI); 933 } 934 935 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 936 const SPIRVType *ResType, 937 MachineInstr &I) const { 938 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 939 report_fatal_error( 940 "llvm.stacksave intrinsic: this instruction requires the following " 941 "SPIR-V extension: SPV_INTEL_variable_length_array", 942 false); 943 MachineBasicBlock &BB = *I.getParent(); 944 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 945 .addDef(ResVReg) 946 .addUse(GR.getSPIRVTypeID(ResType)) 947 .constrainAllUses(TII, TRI, RBI); 948 } 949 950 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 951 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 952 report_fatal_error( 953 "llvm.stackrestore intrinsic: this instruction requires the following " 954 "SPIR-V extension: SPV_INTEL_variable_length_array", 955 false); 956 if (!I.getOperand(0).isReg()) 957 return false; 958 MachineBasicBlock &BB = *I.getParent(); 959 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 960 .addUse(I.getOperand(0).getReg()) 961 .constrainAllUses(TII, TRI, RBI); 962 } 963 964 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 965 MachineInstr &I) const { 966 MachineBasicBlock &BB = *I.getParent(); 967 Register SrcReg = I.getOperand(1).getReg(); 968 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 969 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 970 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 971 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 972 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 973 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 974 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 975 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 976 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 977 // TODO: check if we have such GV, add init, use buildGlobalVariable. 978 Function &CurFunction = GR.CurMF->getFunction(); 979 Type *LLVMArrTy = 980 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 981 // Module takes ownership of the global var. 982 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 983 true, GlobalValue::InternalLinkage, 984 Constant::getNullValue(LLVMArrTy)); 985 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 986 GR.add(GV, GR.CurMF, VarReg); 987 988 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 989 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 990 .addDef(VarReg) 991 .addUse(GR.getSPIRVTypeID(VarTy)) 992 .addImm(SPIRV::StorageClass::UniformConstant) 993 .addUse(Const) 994 .constrainAllUses(TII, TRI, RBI); 995 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 996 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 997 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 998 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 999 } 1000 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1001 .addUse(I.getOperand(0).getReg()) 1002 .addUse(SrcReg) 1003 .addUse(I.getOperand(2).getReg()); 1004 if (I.getNumMemOperands()) 1005 addMemoryOperands(*I.memoperands_begin(), MIB); 1006 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1007 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1008 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1009 .addUse(MIB->getOperand(0).getReg()); 1010 return Result; 1011 } 1012 1013 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1014 const SPIRVType *ResType, 1015 MachineInstr &I, 1016 unsigned NewOpcode, 1017 unsigned NegateOpcode) const { 1018 assert(I.hasOneMemOperand()); 1019 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1020 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1021 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1022 Register ScopeReg = buildI32Constant(Scope, I); 1023 1024 Register Ptr = I.getOperand(1).getReg(); 1025 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1026 // auto ScSem = 1027 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1028 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1029 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1030 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 1031 1032 bool Result = false; 1033 Register ValueReg = I.getOperand(2).getReg(); 1034 if (NegateOpcode != 0) { 1035 // Translation with negative value operand is requested 1036 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1037 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1038 ValueReg = TmpReg; 1039 } 1040 1041 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1042 .addDef(ResVReg) 1043 .addUse(GR.getSPIRVTypeID(ResType)) 1044 .addUse(Ptr) 1045 .addUse(ScopeReg) 1046 .addUse(MemSemReg) 1047 .addUse(ValueReg) 1048 .constrainAllUses(TII, TRI, RBI); 1049 return Result; 1050 } 1051 1052 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1053 unsigned ArgI = I.getNumOperands() - 1; 1054 Register SrcReg = 1055 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1056 SPIRVType *DefType = 1057 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1058 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1059 report_fatal_error( 1060 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1061 1062 SPIRVType *ScalarType = 1063 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1064 MachineBasicBlock &BB = *I.getParent(); 1065 bool Res = false; 1066 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1067 Register ResVReg = I.getOperand(i).getReg(); 1068 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1069 if (!ResType) { 1070 // There was no "assign type" actions, let's fix this now 1071 ResType = ScalarType; 1072 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1073 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1074 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1075 } 1076 auto MIB = 1077 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1078 .addDef(ResVReg) 1079 .addUse(GR.getSPIRVTypeID(ResType)) 1080 .addUse(SrcReg) 1081 .addImm(static_cast<int64_t>(i)); 1082 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1083 } 1084 return Res; 1085 } 1086 1087 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1088 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1089 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1090 Register MemSemReg = buildI32Constant(MemSem, I); 1091 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1092 uint32_t Scope = static_cast<uint32_t>( 1093 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1094 Register ScopeReg = buildI32Constant(Scope, I); 1095 MachineBasicBlock &BB = *I.getParent(); 1096 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1097 .addUse(ScopeReg) 1098 .addUse(MemSemReg) 1099 .constrainAllUses(TII, TRI, RBI); 1100 } 1101 1102 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1103 const SPIRVType *ResType, 1104 MachineInstr &I, 1105 unsigned Opcode) const { 1106 Type *ResTy = nullptr; 1107 StringRef ResName; 1108 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1109 report_fatal_error( 1110 "Not enough info to select the arithmetic with overflow instruction"); 1111 if (!ResTy || !ResTy->isStructTy()) 1112 report_fatal_error("Expect struct type result for the arithmetic " 1113 "with overflow instruction"); 1114 // "Result Type must be from OpTypeStruct. The struct must have two members, 1115 // and the two members must be the same type." 1116 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1117 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1118 // Build SPIR-V types and constant(s) if needed. 1119 MachineIRBuilder MIRBuilder(I); 1120 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1121 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1122 assert(I.getNumDefs() > 1 && "Not enought operands"); 1123 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1124 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1125 if (N > 1) 1126 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1127 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1128 Register ZeroReg = buildZerosVal(ResType, I); 1129 // A new virtual register to store the result struct. 1130 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1131 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1132 // Build the result name if needed. 1133 if (ResName.size() > 0) 1134 buildOpName(StructVReg, ResName, MIRBuilder); 1135 // Build the arithmetic with overflow instruction. 1136 MachineBasicBlock &BB = *I.getParent(); 1137 auto MIB = 1138 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1139 .addDef(StructVReg) 1140 .addUse(GR.getSPIRVTypeID(StructType)); 1141 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1142 MIB.addUse(I.getOperand(i).getReg()); 1143 bool Status = MIB.constrainAllUses(TII, TRI, RBI); 1144 // Build instructions to extract fields of the instruction's result. 1145 // A new virtual register to store the higher part of the result struct. 1146 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1147 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1148 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1149 auto MIB = 1150 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1151 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1152 .addUse(GR.getSPIRVTypeID(ResType)) 1153 .addUse(StructVReg) 1154 .addImm(i); 1155 Status &= MIB.constrainAllUses(TII, TRI, RBI); 1156 } 1157 // Build boolean value from the higher part. 1158 Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1159 .addDef(I.getOperand(1).getReg()) 1160 .addUse(BoolTypeReg) 1161 .addUse(HigherVReg) 1162 .addUse(ZeroReg) 1163 .constrainAllUses(TII, TRI, RBI); 1164 return Status; 1165 } 1166 1167 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1168 const SPIRVType *ResType, 1169 MachineInstr &I) const { 1170 Register ScopeReg; 1171 Register MemSemEqReg; 1172 Register MemSemNeqReg; 1173 Register Ptr = I.getOperand(2).getReg(); 1174 if (!isa<GIntrinsic>(I)) { 1175 assert(I.hasOneMemOperand()); 1176 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1177 unsigned Scope = static_cast<uint32_t>(getMemScope( 1178 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1179 ScopeReg = buildI32Constant(Scope, I); 1180 1181 unsigned ScSem = static_cast<uint32_t>( 1182 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1183 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1184 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1185 MemSemEqReg = buildI32Constant(MemSemEq, I); 1186 AtomicOrdering FO = MemOp->getFailureOrdering(); 1187 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1188 MemSemNeqReg = 1189 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 1190 } else { 1191 ScopeReg = I.getOperand(5).getReg(); 1192 MemSemEqReg = I.getOperand(6).getReg(); 1193 MemSemNeqReg = I.getOperand(7).getReg(); 1194 } 1195 1196 Register Cmp = I.getOperand(3).getReg(); 1197 Register Val = I.getOperand(4).getReg(); 1198 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1199 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1200 const DebugLoc &DL = I.getDebugLoc(); 1201 bool Result = 1202 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1203 .addDef(ACmpRes) 1204 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1205 .addUse(Ptr) 1206 .addUse(ScopeReg) 1207 .addUse(MemSemEqReg) 1208 .addUse(MemSemNeqReg) 1209 .addUse(Val) 1210 .addUse(Cmp) 1211 .constrainAllUses(TII, TRI, RBI); 1212 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1213 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1214 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1215 .addDef(CmpSuccReg) 1216 .addUse(GR.getSPIRVTypeID(BoolTy)) 1217 .addUse(ACmpRes) 1218 .addUse(Cmp) 1219 .constrainAllUses(TII, TRI, RBI); 1220 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1221 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1222 .addDef(TmpReg) 1223 .addUse(GR.getSPIRVTypeID(ResType)) 1224 .addUse(ACmpRes) 1225 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1226 .addImm(0) 1227 .constrainAllUses(TII, TRI, RBI); 1228 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1229 .addDef(ResVReg) 1230 .addUse(GR.getSPIRVTypeID(ResType)) 1231 .addUse(CmpSuccReg) 1232 .addUse(TmpReg) 1233 .addImm(1) 1234 .constrainAllUses(TII, TRI, RBI); 1235 return Result; 1236 } 1237 1238 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1239 switch (SC) { 1240 case SPIRV::StorageClass::Workgroup: 1241 case SPIRV::StorageClass::CrossWorkgroup: 1242 case SPIRV::StorageClass::Function: 1243 return true; 1244 default: 1245 return false; 1246 } 1247 } 1248 1249 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1250 switch (SC) { 1251 case SPIRV::StorageClass::DeviceOnlyINTEL: 1252 case SPIRV::StorageClass::HostOnlyINTEL: 1253 return true; 1254 default: 1255 return false; 1256 } 1257 } 1258 1259 // Returns true ResVReg is referred only from global vars and OpName's. 1260 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1261 bool IsGRef = false; 1262 bool IsAllowedRefs = 1263 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1264 [&IsGRef](auto const &It) { 1265 unsigned Opcode = It.getOpcode(); 1266 if (Opcode == SPIRV::OpConstantComposite || 1267 Opcode == SPIRV::OpVariable || 1268 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1269 return IsGRef = true; 1270 return Opcode == SPIRV::OpName; 1271 }); 1272 return IsAllowedRefs && IsGRef; 1273 } 1274 1275 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1276 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1277 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1278 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1279 } 1280 1281 MachineInstrBuilder 1282 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1283 Register Src, Register DestType, 1284 uint32_t Opcode) const { 1285 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1286 TII.get(SPIRV::OpSpecConstantOp)) 1287 .addDef(Dest) 1288 .addUse(DestType) 1289 .addImm(Opcode) 1290 .addUse(Src); 1291 } 1292 1293 MachineInstrBuilder 1294 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1295 SPIRVType *SrcPtrTy) const { 1296 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1297 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1298 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1299 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1300 SPIRV::StorageClass::Generic), 1301 GR.getPointerSize())); 1302 MachineFunction *MF = I.getParent()->getParent(); 1303 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1304 MachineInstrBuilder MIB = buildSpecConstantOp( 1305 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1306 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1307 GR.add(MIB.getInstr(), MF, Tmp); 1308 return MIB; 1309 } 1310 1311 // In SPIR-V address space casting can only happen to and from the Generic 1312 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1313 // pointers to and from Generic pointers. As such, we can convert e.g. from 1314 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1315 // other combinations can only be done by a bitcast, and are probably not safe. 1316 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1317 const SPIRVType *ResType, 1318 MachineInstr &I) const { 1319 MachineBasicBlock &BB = *I.getParent(); 1320 const DebugLoc &DL = I.getDebugLoc(); 1321 1322 Register SrcPtr = I.getOperand(1).getReg(); 1323 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1324 1325 // don't generate a cast for a null that may be represented by OpTypeInt 1326 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1327 ResType->getOpcode() != SPIRV::OpTypePointer) 1328 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1329 .addDef(ResVReg) 1330 .addUse(SrcPtr) 1331 .constrainAllUses(TII, TRI, RBI); 1332 1333 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1334 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1335 1336 if (isASCastInGVar(MRI, ResVReg)) { 1337 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1338 // are expressed by OpSpecConstantOp with an Opcode. 1339 // TODO: maybe insert a check whether the Kernel capability was declared and 1340 // so PtrCastToGeneric/GenericCastToPtr are available. 1341 unsigned SpecOpcode = 1342 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1343 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1344 : (SrcSC == SPIRV::StorageClass::Generic && 1345 isGenericCastablePtr(DstSC) 1346 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1347 : 0); 1348 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1349 // correct value of ResType and use general i8* instead. Maybe this should 1350 // be addressed in the emit-intrinsic step to infer a correct 1351 // OpConstantComposite type. 1352 if (SpecOpcode) { 1353 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1354 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1355 .constrainAllUses(TII, TRI, RBI); 1356 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1357 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1358 return MIB.constrainAllUses(TII, TRI, RBI) && 1359 buildSpecConstantOp( 1360 I, ResVReg, MIB->getOperand(0).getReg(), 1361 getUcharPtrTypeReg(I, DstSC), 1362 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1363 .constrainAllUses(TII, TRI, RBI); 1364 } 1365 } 1366 1367 // don't generate a cast between identical storage classes 1368 if (SrcSC == DstSC) 1369 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1370 .addDef(ResVReg) 1371 .addUse(SrcPtr) 1372 .constrainAllUses(TII, TRI, RBI); 1373 1374 // Casting from an eligible pointer to Generic. 1375 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1376 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1377 // Casting from Generic to an eligible pointer. 1378 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1379 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1380 // Casting between 2 eligible pointers using Generic as an intermediary. 1381 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1382 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1383 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1384 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1385 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1386 .addDef(Tmp) 1387 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1388 .addUse(SrcPtr) 1389 .constrainAllUses(TII, TRI, RBI); 1390 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1391 .addDef(ResVReg) 1392 .addUse(GR.getSPIRVTypeID(ResType)) 1393 .addUse(Tmp) 1394 .constrainAllUses(TII, TRI, RBI); 1395 } 1396 1397 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1398 // be applied 1399 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1400 return selectUnOp(ResVReg, ResType, I, 1401 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1402 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1403 return selectUnOp(ResVReg, ResType, I, 1404 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1405 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1406 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1407 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1408 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1409 1410 // Bitcast for pointers requires that the address spaces must match 1411 return false; 1412 } 1413 1414 static unsigned getFCmpOpcode(unsigned PredNum) { 1415 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1416 switch (Pred) { 1417 case CmpInst::FCMP_OEQ: 1418 return SPIRV::OpFOrdEqual; 1419 case CmpInst::FCMP_OGE: 1420 return SPIRV::OpFOrdGreaterThanEqual; 1421 case CmpInst::FCMP_OGT: 1422 return SPIRV::OpFOrdGreaterThan; 1423 case CmpInst::FCMP_OLE: 1424 return SPIRV::OpFOrdLessThanEqual; 1425 case CmpInst::FCMP_OLT: 1426 return SPIRV::OpFOrdLessThan; 1427 case CmpInst::FCMP_ONE: 1428 return SPIRV::OpFOrdNotEqual; 1429 case CmpInst::FCMP_ORD: 1430 return SPIRV::OpOrdered; 1431 case CmpInst::FCMP_UEQ: 1432 return SPIRV::OpFUnordEqual; 1433 case CmpInst::FCMP_UGE: 1434 return SPIRV::OpFUnordGreaterThanEqual; 1435 case CmpInst::FCMP_UGT: 1436 return SPIRV::OpFUnordGreaterThan; 1437 case CmpInst::FCMP_ULE: 1438 return SPIRV::OpFUnordLessThanEqual; 1439 case CmpInst::FCMP_ULT: 1440 return SPIRV::OpFUnordLessThan; 1441 case CmpInst::FCMP_UNE: 1442 return SPIRV::OpFUnordNotEqual; 1443 case CmpInst::FCMP_UNO: 1444 return SPIRV::OpUnordered; 1445 default: 1446 llvm_unreachable("Unknown predicate type for FCmp"); 1447 } 1448 } 1449 1450 static unsigned getICmpOpcode(unsigned PredNum) { 1451 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1452 switch (Pred) { 1453 case CmpInst::ICMP_EQ: 1454 return SPIRV::OpIEqual; 1455 case CmpInst::ICMP_NE: 1456 return SPIRV::OpINotEqual; 1457 case CmpInst::ICMP_SGE: 1458 return SPIRV::OpSGreaterThanEqual; 1459 case CmpInst::ICMP_SGT: 1460 return SPIRV::OpSGreaterThan; 1461 case CmpInst::ICMP_SLE: 1462 return SPIRV::OpSLessThanEqual; 1463 case CmpInst::ICMP_SLT: 1464 return SPIRV::OpSLessThan; 1465 case CmpInst::ICMP_UGE: 1466 return SPIRV::OpUGreaterThanEqual; 1467 case CmpInst::ICMP_UGT: 1468 return SPIRV::OpUGreaterThan; 1469 case CmpInst::ICMP_ULE: 1470 return SPIRV::OpULessThanEqual; 1471 case CmpInst::ICMP_ULT: 1472 return SPIRV::OpULessThan; 1473 default: 1474 llvm_unreachable("Unknown predicate type for ICmp"); 1475 } 1476 } 1477 1478 static unsigned getPtrCmpOpcode(unsigned Pred) { 1479 switch (static_cast<CmpInst::Predicate>(Pred)) { 1480 case CmpInst::ICMP_EQ: 1481 return SPIRV::OpPtrEqual; 1482 case CmpInst::ICMP_NE: 1483 return SPIRV::OpPtrNotEqual; 1484 default: 1485 llvm_unreachable("Unknown predicate type for pointer comparison"); 1486 } 1487 } 1488 1489 // Return the logical operation, or abort if none exists. 1490 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1491 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1492 switch (Pred) { 1493 case CmpInst::ICMP_EQ: 1494 return SPIRV::OpLogicalEqual; 1495 case CmpInst::ICMP_NE: 1496 return SPIRV::OpLogicalNotEqual; 1497 default: 1498 llvm_unreachable("Unknown predicate type for Bool comparison"); 1499 } 1500 } 1501 1502 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1503 if (!LLVMFloatTy) 1504 return APFloat::getZero(APFloat::IEEEsingle()); 1505 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1506 case Type::HalfTyID: 1507 return APFloat::getZero(APFloat::IEEEhalf()); 1508 default: 1509 case Type::FloatTyID: 1510 return APFloat::getZero(APFloat::IEEEsingle()); 1511 case Type::DoubleTyID: 1512 return APFloat::getZero(APFloat::IEEEdouble()); 1513 } 1514 } 1515 1516 static APFloat getOneFP(const Type *LLVMFloatTy) { 1517 if (!LLVMFloatTy) 1518 return APFloat::getOne(APFloat::IEEEsingle()); 1519 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1520 case Type::HalfTyID: 1521 return APFloat::getOne(APFloat::IEEEhalf()); 1522 default: 1523 case Type::FloatTyID: 1524 return APFloat::getOne(APFloat::IEEEsingle()); 1525 case Type::DoubleTyID: 1526 return APFloat::getOne(APFloat::IEEEdouble()); 1527 } 1528 } 1529 1530 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1531 const SPIRVType *ResType, 1532 MachineInstr &I, 1533 unsigned OpAnyOrAll) const { 1534 assert(I.getNumOperands() == 3); 1535 assert(I.getOperand(2).isReg()); 1536 MachineBasicBlock &BB = *I.getParent(); 1537 Register InputRegister = I.getOperand(2).getReg(); 1538 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1539 1540 if (!InputType) 1541 report_fatal_error("Input Type could not be determined."); 1542 1543 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1544 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1545 if (IsBoolTy && !IsVectorTy) { 1546 assert(ResVReg == I.getOperand(0).getReg()); 1547 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1548 TII.get(TargetOpcode::COPY)) 1549 .addDef(ResVReg) 1550 .addUse(InputRegister) 1551 .constrainAllUses(TII, TRI, RBI); 1552 } 1553 1554 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1555 unsigned SpirvNotEqualId = 1556 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1557 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1558 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1559 Register NotEqualReg = ResVReg; 1560 1561 if (IsVectorTy) { 1562 NotEqualReg = IsBoolTy ? InputRegister 1563 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1564 const unsigned NumElts = InputType->getOperand(2).getImm(); 1565 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1566 } 1567 1568 if (!IsBoolTy) { 1569 Register ConstZeroReg = 1570 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1571 1572 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1573 .addDef(NotEqualReg) 1574 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1575 .addUse(InputRegister) 1576 .addUse(ConstZeroReg) 1577 .constrainAllUses(TII, TRI, RBI); 1578 } 1579 1580 if (!IsVectorTy) 1581 return true; 1582 1583 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1584 .addDef(ResVReg) 1585 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1586 .addUse(NotEqualReg) 1587 .constrainAllUses(TII, TRI, RBI); 1588 } 1589 1590 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1591 const SPIRVType *ResType, 1592 MachineInstr &I) const { 1593 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1594 } 1595 1596 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1597 const SPIRVType *ResType, 1598 MachineInstr &I) const { 1599 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1600 } 1601 1602 // Select the OpDot instruction for the given float dot 1603 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1604 const SPIRVType *ResType, 1605 MachineInstr &I) const { 1606 assert(I.getNumOperands() == 4); 1607 assert(I.getOperand(2).isReg()); 1608 assert(I.getOperand(3).isReg()); 1609 1610 [[maybe_unused]] SPIRVType *VecType = 1611 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1612 1613 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1614 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1615 "dot product requires a vector of at least 2 components"); 1616 1617 [[maybe_unused]] SPIRVType *EltType = 1618 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1619 1620 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1621 1622 MachineBasicBlock &BB = *I.getParent(); 1623 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1624 .addDef(ResVReg) 1625 .addUse(GR.getSPIRVTypeID(ResType)) 1626 .addUse(I.getOperand(2).getReg()) 1627 .addUse(I.getOperand(3).getReg()) 1628 .constrainAllUses(TII, TRI, RBI); 1629 } 1630 1631 // Since pre-1.6 SPIRV has no integer dot implementation, 1632 // expand by piecewise multiplying and adding the results 1633 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1634 const SPIRVType *ResType, 1635 MachineInstr &I) const { 1636 assert(I.getNumOperands() == 4); 1637 assert(I.getOperand(2).isReg()); 1638 assert(I.getOperand(3).isReg()); 1639 MachineBasicBlock &BB = *I.getParent(); 1640 1641 // Multiply the vectors, then sum the results 1642 Register Vec0 = I.getOperand(2).getReg(); 1643 Register Vec1 = I.getOperand(3).getReg(); 1644 Register TmpVec = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1645 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1646 1647 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1648 .addDef(TmpVec) 1649 .addUse(GR.getSPIRVTypeID(VecType)) 1650 .addUse(Vec0) 1651 .addUse(Vec1) 1652 .constrainAllUses(TII, TRI, RBI); 1653 1654 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1655 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1656 "dot product requires a vector of at least 2 components"); 1657 1658 Register Res = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1659 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1660 .addDef(Res) 1661 .addUse(GR.getSPIRVTypeID(ResType)) 1662 .addUse(TmpVec) 1663 .addImm(0) 1664 .constrainAllUses(TII, TRI, RBI); 1665 1666 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1667 Register Elt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1668 1669 Result |= 1670 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1671 .addDef(Elt) 1672 .addUse(GR.getSPIRVTypeID(ResType)) 1673 .addUse(TmpVec) 1674 .addImm(i) 1675 .constrainAllUses(TII, TRI, RBI); 1676 1677 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1678 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1679 : ResVReg; 1680 1681 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1682 .addDef(Sum) 1683 .addUse(GR.getSPIRVTypeID(ResType)) 1684 .addUse(Res) 1685 .addUse(Elt) 1686 .constrainAllUses(TII, TRI, RBI); 1687 Res = Sum; 1688 } 1689 1690 return Result; 1691 } 1692 1693 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1694 /// does not have a saturate builtin. 1695 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1696 const SPIRVType *ResType, 1697 MachineInstr &I) const { 1698 assert(I.getNumOperands() == 3); 1699 assert(I.getOperand(2).isReg()); 1700 MachineBasicBlock &BB = *I.getParent(); 1701 Register VZero = buildZerosValF(ResType, I); 1702 Register VOne = buildOnesValF(ResType, I); 1703 1704 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1705 .addDef(ResVReg) 1706 .addUse(GR.getSPIRVTypeID(ResType)) 1707 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1708 .addImm(GL::FClamp) 1709 .addUse(I.getOperand(2).getReg()) 1710 .addUse(VZero) 1711 .addUse(VOne) 1712 .constrainAllUses(TII, TRI, RBI); 1713 } 1714 1715 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1716 const SPIRVType *ResType, 1717 MachineInstr &I) const { 1718 assert(I.getNumOperands() == 3); 1719 assert(I.getOperand(2).isReg()); 1720 MachineBasicBlock &BB = *I.getParent(); 1721 Register InputRegister = I.getOperand(2).getReg(); 1722 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1723 auto &DL = I.getDebugLoc(); 1724 1725 if (!InputType) 1726 report_fatal_error("Input Type could not be determined."); 1727 1728 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1729 1730 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1731 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1732 1733 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1734 1735 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1736 Register SignReg = NeedsConversion 1737 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1738 : ResVReg; 1739 1740 bool Result = 1741 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1742 .addDef(SignReg) 1743 .addUse(GR.getSPIRVTypeID(InputType)) 1744 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1745 .addImm(SignOpcode) 1746 .addUse(InputRegister) 1747 .constrainAllUses(TII, TRI, RBI); 1748 1749 if (NeedsConversion) { 1750 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1751 Result |= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1752 .addDef(ResVReg) 1753 .addUse(GR.getSPIRVTypeID(ResType)) 1754 .addUse(SignReg) 1755 .constrainAllUses(TII, TRI, RBI); 1756 } 1757 1758 return Result; 1759 } 1760 1761 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1762 const SPIRVType *ResType, 1763 MachineInstr &I) const { 1764 MachineBasicBlock &BB = *I.getParent(); 1765 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1766 .addDef(ResVReg) 1767 .addUse(GR.getSPIRVTypeID(ResType)) 1768 .addUse(I.getOperand(1).getReg()) 1769 .constrainAllUses(TII, TRI, RBI); 1770 } 1771 1772 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1773 const SPIRVType *ResType, 1774 MachineInstr &I) const { 1775 // There is no way to implement `freeze` correctly without support on SPIR-V 1776 // standard side, but we may at least address a simple (static) case when 1777 // undef/poison value presence is obvious. The main benefit of even 1778 // incomplete `freeze` support is preventing of translation from crashing due 1779 // to lack of support on legalization and instruction selection steps. 1780 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1781 return false; 1782 Register OpReg = I.getOperand(1).getReg(); 1783 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1784 Register Reg; 1785 switch (Def->getOpcode()) { 1786 case SPIRV::ASSIGN_TYPE: 1787 if (MachineInstr *AssignToDef = 1788 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1789 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1790 Reg = Def->getOperand(2).getReg(); 1791 } 1792 break; 1793 case SPIRV::OpUndef: 1794 Reg = Def->getOperand(1).getReg(); 1795 break; 1796 } 1797 unsigned DestOpCode; 1798 if (Reg.isValid()) { 1799 DestOpCode = SPIRV::OpConstantNull; 1800 } else { 1801 DestOpCode = TargetOpcode::COPY; 1802 Reg = OpReg; 1803 } 1804 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1805 .addDef(I.getOperand(0).getReg()) 1806 .addUse(Reg) 1807 .constrainAllUses(TII, TRI, RBI); 1808 } 1809 return false; 1810 } 1811 1812 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1813 const SPIRVType *ResType) { 1814 Register OpReg = ResType->getOperand(2).getReg(); 1815 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1816 if (!OpDef) 1817 return 0; 1818 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1819 OpDef->getOperand(1).isReg()) { 1820 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1821 OpDef = RefDef; 1822 } 1823 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 1824 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 1825 : 0; 1826 return N; 1827 } 1828 1829 // Return true if the type represents a constant register 1830 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 1831 SmallPtrSet<SPIRVType *, 4> &Visited) { 1832 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1833 OpDef->getOperand(1).isReg()) { 1834 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1835 OpDef = RefDef; 1836 } 1837 1838 if (Visited.contains(OpDef)) 1839 return true; 1840 Visited.insert(OpDef); 1841 1842 unsigned Opcode = OpDef->getOpcode(); 1843 switch (Opcode) { 1844 case TargetOpcode::G_CONSTANT: 1845 case TargetOpcode::G_FCONSTANT: 1846 return true; 1847 case TargetOpcode::G_INTRINSIC: 1848 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 1849 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 1850 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 1851 Intrinsic::spv_const_composite; 1852 case TargetOpcode::G_BUILD_VECTOR: 1853 case TargetOpcode::G_SPLAT_VECTOR: { 1854 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 1855 i++) { 1856 SPIRVType *OpNestedDef = 1857 OpDef->getOperand(i).isReg() 1858 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 1859 : nullptr; 1860 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 1861 return false; 1862 } 1863 return true; 1864 } 1865 } 1866 return false; 1867 } 1868 1869 // Return true if the virtual register represents a constant 1870 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 1871 SmallPtrSet<SPIRVType *, 4> Visited; 1872 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 1873 return isConstReg(MRI, OpDef, Visited); 1874 return false; 1875 } 1876 1877 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 1878 const SPIRVType *ResType, 1879 MachineInstr &I) const { 1880 unsigned N = 0; 1881 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1882 N = GR.getScalarOrVectorComponentCount(ResType); 1883 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1884 N = getArrayComponentCount(MRI, ResType); 1885 else 1886 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 1887 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 1888 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 1889 1890 // check if we may construct a constant vector 1891 bool IsConst = true; 1892 for (unsigned i = I.getNumExplicitDefs(); 1893 i < I.getNumExplicitOperands() && IsConst; ++i) 1894 if (!isConstReg(MRI, I.getOperand(i).getReg())) 1895 IsConst = false; 1896 1897 if (!IsConst && N < 2) 1898 report_fatal_error( 1899 "There must be at least two constituent operands in a vector"); 1900 1901 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1902 TII.get(IsConst ? SPIRV::OpConstantComposite 1903 : SPIRV::OpCompositeConstruct)) 1904 .addDef(ResVReg) 1905 .addUse(GR.getSPIRVTypeID(ResType)); 1906 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 1907 MIB.addUse(I.getOperand(i).getReg()); 1908 return MIB.constrainAllUses(TII, TRI, RBI); 1909 } 1910 1911 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 1912 const SPIRVType *ResType, 1913 MachineInstr &I) const { 1914 unsigned N = 0; 1915 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1916 N = GR.getScalarOrVectorComponentCount(ResType); 1917 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1918 N = getArrayComponentCount(MRI, ResType); 1919 else 1920 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 1921 1922 unsigned OpIdx = I.getNumExplicitDefs(); 1923 if (!I.getOperand(OpIdx).isReg()) 1924 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 1925 1926 // check if we may construct a constant vector 1927 Register OpReg = I.getOperand(OpIdx).getReg(); 1928 bool IsConst = isConstReg(MRI, OpReg); 1929 1930 if (!IsConst && N < 2) 1931 report_fatal_error( 1932 "There must be at least two constituent operands in a vector"); 1933 1934 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1935 TII.get(IsConst ? SPIRV::OpConstantComposite 1936 : SPIRV::OpCompositeConstruct)) 1937 .addDef(ResVReg) 1938 .addUse(GR.getSPIRVTypeID(ResType)); 1939 for (unsigned i = 0; i < N; ++i) 1940 MIB.addUse(OpReg); 1941 return MIB.constrainAllUses(TII, TRI, RBI); 1942 } 1943 1944 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 1945 const SPIRVType *ResType, 1946 unsigned CmpOpc, 1947 MachineInstr &I) const { 1948 Register Cmp0 = I.getOperand(2).getReg(); 1949 Register Cmp1 = I.getOperand(3).getReg(); 1950 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1951 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1952 "CMP operands should have the same type"); 1953 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1954 .addDef(ResVReg) 1955 .addUse(GR.getSPIRVTypeID(ResType)) 1956 .addUse(Cmp0) 1957 .addUse(Cmp1) 1958 .constrainAllUses(TII, TRI, RBI); 1959 } 1960 1961 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1962 const SPIRVType *ResType, 1963 MachineInstr &I) const { 1964 auto Pred = I.getOperand(1).getPredicate(); 1965 unsigned CmpOpc; 1966 1967 Register CmpOperand = I.getOperand(2).getReg(); 1968 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1969 CmpOpc = getPtrCmpOpcode(Pred); 1970 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1971 CmpOpc = getBoolCmpOpcode(Pred); 1972 else 1973 CmpOpc = getICmpOpcode(Pred); 1974 return selectCmp(ResVReg, ResType, CmpOpc, I); 1975 } 1976 1977 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 1978 const MachineInstr &I, 1979 int OpIdx) const { 1980 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1981 "Expected G_FCONSTANT"); 1982 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1983 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1984 } 1985 1986 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1987 const MachineInstr &I, 1988 int OpIdx) const { 1989 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1990 "Expected G_CONSTANT"); 1991 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1992 } 1993 1994 Register 1995 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1996 const SPIRVType *ResType) const { 1997 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1998 const SPIRVType *SpvI32Ty = 1999 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2000 // Find a constant in DT or build a new one. 2001 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2002 Register NewReg = GR.find(ConstInt, GR.CurMF); 2003 if (!NewReg.isValid()) { 2004 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2005 GR.add(ConstInt, GR.CurMF, NewReg); 2006 MachineInstr *MI; 2007 MachineBasicBlock &BB = *I.getParent(); 2008 if (Val == 0) { 2009 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2010 .addDef(NewReg) 2011 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2012 } else { 2013 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2014 .addDef(NewReg) 2015 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2016 .addImm(APInt(32, Val).getZExtValue()); 2017 } 2018 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2019 } 2020 return NewReg; 2021 } 2022 2023 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2024 const SPIRVType *ResType, 2025 MachineInstr &I) const { 2026 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2027 return selectCmp(ResVReg, ResType, CmpOp, I); 2028 } 2029 2030 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2031 MachineInstr &I) const { 2032 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2033 bool ZeroAsNull = STI.isOpenCLEnv(); 2034 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2035 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2036 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2037 } 2038 2039 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2040 MachineInstr &I) const { 2041 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2042 bool ZeroAsNull = STI.isOpenCLEnv(); 2043 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2044 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2045 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2046 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2047 } 2048 2049 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2050 MachineInstr &I) const { 2051 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2052 bool ZeroAsNull = STI.isOpenCLEnv(); 2053 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2054 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2055 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2056 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2057 } 2058 2059 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2060 const SPIRVType *ResType, 2061 MachineInstr &I) const { 2062 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2063 APInt One = 2064 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2065 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2066 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2067 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2068 } 2069 2070 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2071 const SPIRVType *ResType, 2072 MachineInstr &I, 2073 bool IsSigned) const { 2074 // To extend a bool, we need to use OpSelect between constants. 2075 Register ZeroReg = buildZerosVal(ResType, I); 2076 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2077 bool IsScalarBool = 2078 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2079 unsigned Opcode = 2080 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2081 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2082 .addDef(ResVReg) 2083 .addUse(GR.getSPIRVTypeID(ResType)) 2084 .addUse(I.getOperand(1).getReg()) 2085 .addUse(OneReg) 2086 .addUse(ZeroReg) 2087 .constrainAllUses(TII, TRI, RBI); 2088 } 2089 2090 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2091 const SPIRVType *ResType, 2092 MachineInstr &I, bool IsSigned, 2093 unsigned Opcode) const { 2094 Register SrcReg = I.getOperand(1).getReg(); 2095 // We can convert bool value directly to float type without OpConvert*ToF, 2096 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2097 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2098 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2099 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2100 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2101 const unsigned NumElts = ResType->getOperand(2).getImm(); 2102 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2103 } 2104 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2105 selectSelect(SrcReg, TmpType, I, false); 2106 } 2107 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2108 } 2109 2110 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2111 const SPIRVType *ResType, 2112 MachineInstr &I, bool IsSigned) const { 2113 Register SrcReg = I.getOperand(1).getReg(); 2114 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2115 return selectSelect(ResVReg, ResType, I, IsSigned); 2116 2117 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2118 if (SrcType == ResType) { 2119 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2120 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2121 if (DstRC != SrcRC && SrcRC) 2122 MRI->setRegClass(ResVReg, SrcRC); 2123 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2124 TII.get(TargetOpcode::COPY)) 2125 .addDef(ResVReg) 2126 .addUse(SrcReg) 2127 .constrainAllUses(TII, TRI, RBI); 2128 } 2129 2130 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2131 return selectUnOp(ResVReg, ResType, I, Opcode); 2132 } 2133 2134 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2135 Register ResVReg, 2136 MachineInstr &I, 2137 const SPIRVType *IntTy, 2138 const SPIRVType *BoolTy) const { 2139 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2140 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2141 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2142 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2143 Register Zero = buildZerosVal(IntTy, I); 2144 Register One = buildOnesVal(false, IntTy, I); 2145 MachineBasicBlock &BB = *I.getParent(); 2146 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2147 .addDef(BitIntReg) 2148 .addUse(GR.getSPIRVTypeID(IntTy)) 2149 .addUse(IntReg) 2150 .addUse(One) 2151 .constrainAllUses(TII, TRI, RBI); 2152 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2153 .addDef(ResVReg) 2154 .addUse(GR.getSPIRVTypeID(BoolTy)) 2155 .addUse(BitIntReg) 2156 .addUse(Zero) 2157 .constrainAllUses(TII, TRI, RBI); 2158 } 2159 2160 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2161 const SPIRVType *ResType, 2162 MachineInstr &I) const { 2163 Register IntReg = I.getOperand(1).getReg(); 2164 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2165 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2166 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2167 if (ArgType == ResType) { 2168 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2169 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2170 if (DstRC != SrcRC && SrcRC) 2171 MRI->setRegClass(ResVReg, SrcRC); 2172 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2173 TII.get(TargetOpcode::COPY)) 2174 .addDef(ResVReg) 2175 .addUse(IntReg) 2176 .constrainAllUses(TII, TRI, RBI); 2177 } 2178 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2179 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2180 return selectUnOp(ResVReg, ResType, I, Opcode); 2181 } 2182 2183 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2184 const SPIRVType *ResType, 2185 const APInt &Imm, 2186 MachineInstr &I) const { 2187 unsigned TyOpcode = ResType->getOpcode(); 2188 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2189 MachineBasicBlock &BB = *I.getParent(); 2190 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2191 Imm.isZero()) 2192 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2193 .addDef(ResVReg) 2194 .addUse(GR.getSPIRVTypeID(ResType)) 2195 .constrainAllUses(TII, TRI, RBI); 2196 if (TyOpcode == SPIRV::OpTypeInt) { 2197 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2198 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2199 if (Reg == ResVReg) 2200 return true; 2201 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2202 .addDef(ResVReg) 2203 .addUse(Reg) 2204 .constrainAllUses(TII, TRI, RBI); 2205 } 2206 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2207 .addDef(ResVReg) 2208 .addUse(GR.getSPIRVTypeID(ResType)); 2209 // <=32-bit integers should be caught by the sdag pattern. 2210 assert(Imm.getBitWidth() > 32); 2211 addNumImm(Imm, MIB); 2212 return MIB.constrainAllUses(TII, TRI, RBI); 2213 } 2214 2215 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2216 const SPIRVType *ResType, 2217 MachineInstr &I) const { 2218 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2219 .addDef(ResVReg) 2220 .addUse(GR.getSPIRVTypeID(ResType)) 2221 .constrainAllUses(TII, TRI, RBI); 2222 } 2223 2224 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2225 assert(MO.isReg()); 2226 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2227 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2228 assert(TypeInst->getOperand(1).isReg()); 2229 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2230 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2231 } 2232 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2233 } 2234 2235 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2236 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2237 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2238 return TypeInst->getOperand(2).getImm(); 2239 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2240 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2241 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2242 } 2243 2244 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2245 const SPIRVType *ResType, 2246 MachineInstr &I) const { 2247 MachineBasicBlock &BB = *I.getParent(); 2248 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2249 .addDef(ResVReg) 2250 .addUse(GR.getSPIRVTypeID(ResType)) 2251 // object to insert 2252 .addUse(I.getOperand(3).getReg()) 2253 // composite to insert into 2254 .addUse(I.getOperand(2).getReg()); 2255 for (unsigned i = 4; i < I.getNumOperands(); i++) 2256 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2257 return MIB.constrainAllUses(TII, TRI, RBI); 2258 } 2259 2260 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2261 const SPIRVType *ResType, 2262 MachineInstr &I) const { 2263 MachineBasicBlock &BB = *I.getParent(); 2264 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2265 .addDef(ResVReg) 2266 .addUse(GR.getSPIRVTypeID(ResType)) 2267 .addUse(I.getOperand(2).getReg()); 2268 for (unsigned i = 3; i < I.getNumOperands(); i++) 2269 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2270 return MIB.constrainAllUses(TII, TRI, RBI); 2271 } 2272 2273 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2274 const SPIRVType *ResType, 2275 MachineInstr &I) const { 2276 if (isImm(I.getOperand(4), MRI)) 2277 return selectInsertVal(ResVReg, ResType, I); 2278 MachineBasicBlock &BB = *I.getParent(); 2279 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2280 .addDef(ResVReg) 2281 .addUse(GR.getSPIRVTypeID(ResType)) 2282 .addUse(I.getOperand(2).getReg()) 2283 .addUse(I.getOperand(3).getReg()) 2284 .addUse(I.getOperand(4).getReg()) 2285 .constrainAllUses(TII, TRI, RBI); 2286 } 2287 2288 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2289 const SPIRVType *ResType, 2290 MachineInstr &I) const { 2291 if (isImm(I.getOperand(3), MRI)) 2292 return selectExtractVal(ResVReg, ResType, I); 2293 MachineBasicBlock &BB = *I.getParent(); 2294 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2295 .addDef(ResVReg) 2296 .addUse(GR.getSPIRVTypeID(ResType)) 2297 .addUse(I.getOperand(2).getReg()) 2298 .addUse(I.getOperand(3).getReg()) 2299 .constrainAllUses(TII, TRI, RBI); 2300 } 2301 2302 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2303 const SPIRVType *ResType, 2304 MachineInstr &I) const { 2305 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2306 2307 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2308 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2309 // we have to use Op[InBounds]AccessChain. 2310 const unsigned Opcode = STI.isVulkanEnv() 2311 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2312 : SPIRV::OpAccessChain) 2313 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2314 : SPIRV::OpPtrAccessChain); 2315 2316 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2317 .addDef(ResVReg) 2318 .addUse(GR.getSPIRVTypeID(ResType)) 2319 // Object to get a pointer to. 2320 .addUse(I.getOperand(3).getReg()); 2321 // Adding indices. 2322 const unsigned StartingIndex = 2323 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2324 ? 5 2325 : 4; 2326 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2327 Res.addUse(I.getOperand(i).getReg()); 2328 return Res.constrainAllUses(TII, TRI, RBI); 2329 } 2330 2331 // Maybe wrap a value into OpSpecConstantOp 2332 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2333 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2334 bool Result = true; 2335 unsigned Lim = I.getNumExplicitOperands(); 2336 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2337 Register OpReg = I.getOperand(i).getReg(); 2338 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2339 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2340 SmallPtrSet<SPIRVType *, 4> Visited; 2341 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2342 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2343 GR.isAggregateType(OpType)) { 2344 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2345 // by selectAddrSpaceCast() 2346 CompositeArgs.push_back(OpReg); 2347 continue; 2348 } 2349 MachineFunction *MF = I.getMF(); 2350 Register WrapReg = GR.find(OpDefine, MF); 2351 if (WrapReg.isValid()) { 2352 CompositeArgs.push_back(WrapReg); 2353 continue; 2354 } 2355 // Create a new register for the wrapper 2356 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2357 GR.add(OpDefine, MF, WrapReg); 2358 CompositeArgs.push_back(WrapReg); 2359 // Decorate the wrapper register and generate a new instruction 2360 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2361 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2362 MachineBasicBlock &BB = *I.getParent(); 2363 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2364 .addDef(WrapReg) 2365 .addUse(GR.getSPIRVTypeID(OpType)) 2366 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2367 .addUse(OpReg) 2368 .constrainAllUses(TII, TRI, RBI); 2369 if (!Result) 2370 break; 2371 } 2372 return Result; 2373 } 2374 2375 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2376 const SPIRVType *ResType, 2377 MachineInstr &I) const { 2378 MachineBasicBlock &BB = *I.getParent(); 2379 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2380 switch (IID) { 2381 case Intrinsic::spv_load: 2382 return selectLoad(ResVReg, ResType, I); 2383 case Intrinsic::spv_store: 2384 return selectStore(I); 2385 case Intrinsic::spv_extractv: 2386 return selectExtractVal(ResVReg, ResType, I); 2387 case Intrinsic::spv_insertv: 2388 return selectInsertVal(ResVReg, ResType, I); 2389 case Intrinsic::spv_extractelt: 2390 return selectExtractElt(ResVReg, ResType, I); 2391 case Intrinsic::spv_insertelt: 2392 return selectInsertElt(ResVReg, ResType, I); 2393 case Intrinsic::spv_gep: 2394 return selectGEP(ResVReg, ResType, I); 2395 case Intrinsic::spv_unref_global: 2396 case Intrinsic::spv_init_global: { 2397 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2398 MachineInstr *Init = I.getNumExplicitOperands() > 2 2399 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2400 : nullptr; 2401 assert(MI); 2402 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2403 } 2404 case Intrinsic::spv_undef: { 2405 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2406 .addDef(ResVReg) 2407 .addUse(GR.getSPIRVTypeID(ResType)); 2408 return MIB.constrainAllUses(TII, TRI, RBI); 2409 } 2410 case Intrinsic::spv_const_composite: { 2411 // If no values are attached, the composite is null constant. 2412 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2413 // Select a proper instruction. 2414 unsigned Opcode = SPIRV::OpConstantNull; 2415 SmallVector<Register> CompositeArgs; 2416 if (!IsNull) { 2417 Opcode = SPIRV::OpConstantComposite; 2418 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2419 return false; 2420 } 2421 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2422 .addDef(ResVReg) 2423 .addUse(GR.getSPIRVTypeID(ResType)); 2424 // skip type MD node we already used when generated assign.type for this 2425 if (!IsNull) { 2426 for (Register OpReg : CompositeArgs) 2427 MIB.addUse(OpReg); 2428 } 2429 return MIB.constrainAllUses(TII, TRI, RBI); 2430 } 2431 case Intrinsic::spv_assign_name: { 2432 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2433 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2434 for (unsigned i = I.getNumExplicitDefs() + 2; 2435 i < I.getNumExplicitOperands(); ++i) { 2436 MIB.addImm(I.getOperand(i).getImm()); 2437 } 2438 return MIB.constrainAllUses(TII, TRI, RBI); 2439 } 2440 case Intrinsic::spv_switch: { 2441 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2442 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2443 if (I.getOperand(i).isReg()) 2444 MIB.addReg(I.getOperand(i).getReg()); 2445 else if (I.getOperand(i).isCImm()) 2446 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2447 else if (I.getOperand(i).isMBB()) 2448 MIB.addMBB(I.getOperand(i).getMBB()); 2449 else 2450 llvm_unreachable("Unexpected OpSwitch operand"); 2451 } 2452 return MIB.constrainAllUses(TII, TRI, RBI); 2453 } 2454 case Intrinsic::spv_loop_merge: 2455 case Intrinsic::spv_selection_merge: { 2456 const auto Opcode = IID == Intrinsic::spv_selection_merge 2457 ? SPIRV::OpSelectionMerge 2458 : SPIRV::OpLoopMerge; 2459 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2460 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2461 assert(I.getOperand(i).isMBB()); 2462 MIB.addMBB(I.getOperand(i).getMBB()); 2463 } 2464 MIB.addImm(SPIRV::SelectionControl::None); 2465 return MIB.constrainAllUses(TII, TRI, RBI); 2466 } 2467 case Intrinsic::spv_cmpxchg: 2468 return selectAtomicCmpXchg(ResVReg, ResType, I); 2469 case Intrinsic::spv_unreachable: 2470 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 2471 break; 2472 case Intrinsic::spv_alloca: 2473 return selectFrameIndex(ResVReg, ResType, I); 2474 case Intrinsic::spv_alloca_array: 2475 return selectAllocaArray(ResVReg, ResType, I); 2476 case Intrinsic::spv_assume: 2477 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2478 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2479 .addUse(I.getOperand(1).getReg()); 2480 break; 2481 case Intrinsic::spv_expect: 2482 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2483 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2484 .addDef(ResVReg) 2485 .addUse(GR.getSPIRVTypeID(ResType)) 2486 .addUse(I.getOperand(2).getReg()) 2487 .addUse(I.getOperand(3).getReg()); 2488 break; 2489 case Intrinsic::arithmetic_fence: 2490 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2491 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT)) 2492 .addDef(ResVReg) 2493 .addUse(GR.getSPIRVTypeID(ResType)) 2494 .addUse(I.getOperand(2).getReg()); 2495 else 2496 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 2497 .addUse(I.getOperand(2).getReg()); 2498 break; 2499 case Intrinsic::spv_thread_id: 2500 return selectSpvThreadId(ResVReg, ResType, I); 2501 case Intrinsic::spv_fdot: 2502 return selectFloatDot(ResVReg, ResType, I); 2503 case Intrinsic::spv_udot: 2504 case Intrinsic::spv_sdot: 2505 return selectIntegerDot(ResVReg, ResType, I); 2506 case Intrinsic::spv_all: 2507 return selectAll(ResVReg, ResType, I); 2508 case Intrinsic::spv_any: 2509 return selectAny(ResVReg, ResType, I); 2510 case Intrinsic::spv_cross: 2511 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2512 case Intrinsic::spv_lerp: 2513 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2514 case Intrinsic::spv_length: 2515 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2516 case Intrinsic::spv_degrees: 2517 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2518 case Intrinsic::spv_frac: 2519 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2520 case Intrinsic::spv_normalize: 2521 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2522 case Intrinsic::spv_rsqrt: 2523 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2524 case Intrinsic::spv_sign: 2525 return selectSign(ResVReg, ResType, I); 2526 case Intrinsic::spv_lifetime_start: 2527 case Intrinsic::spv_lifetime_end: { 2528 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2529 : SPIRV::OpLifetimeStop; 2530 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2531 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2532 if (Size == -1) 2533 Size = 0; 2534 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 2535 } break; 2536 case Intrinsic::spv_saturate: 2537 return selectSaturate(ResVReg, ResType, I); 2538 case Intrinsic::spv_wave_is_first_lane: { 2539 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2540 return BuildMI(BB, I, I.getDebugLoc(), 2541 TII.get(SPIRV::OpGroupNonUniformElect)) 2542 .addDef(ResVReg) 2543 .addUse(GR.getSPIRVTypeID(ResType)) 2544 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2545 } 2546 case Intrinsic::spv_step: 2547 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2548 case Intrinsic::spv_radians: 2549 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2550 // Discard intrinsics which we do not expect to actually represent code after 2551 // lowering or intrinsics which are not implemented but should not crash when 2552 // found in a customer's LLVM IR input. 2553 case Intrinsic::instrprof_increment: 2554 case Intrinsic::instrprof_increment_step: 2555 case Intrinsic::instrprof_value_profile: 2556 break; 2557 // Discard internal intrinsics. 2558 case Intrinsic::spv_value_md: 2559 break; 2560 case Intrinsic::spv_handle_fromBinding: { 2561 selectHandleFromBinding(ResVReg, ResType, I); 2562 return true; 2563 } 2564 default: { 2565 std::string DiagMsg; 2566 raw_string_ostream OS(DiagMsg); 2567 I.print(OS); 2568 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2569 report_fatal_error(DiagMsg.c_str(), false); 2570 } 2571 } 2572 return true; 2573 } 2574 2575 void SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2576 const SPIRVType *ResType, 2577 MachineInstr &I) const { 2578 2579 uint32_t Set = foldImm(I.getOperand(2), MRI); 2580 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2581 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2582 2583 MachineIRBuilder MIRBuilder(I); 2584 Register VarReg = 2585 buildPointerToResource(ResType, Set, Binding, ArraySize, MIRBuilder); 2586 2587 // TODO: For now we assume the resource is an image, which needs to be 2588 // loaded to get the handle. That will not be true for storage buffers. 2589 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2590 .addDef(ResVReg) 2591 .addUse(GR.getSPIRVTypeID(ResType)) 2592 .addUse(VarReg); 2593 } 2594 2595 Register SPIRVInstructionSelector::buildPointerToResource( 2596 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 2597 uint32_t ArraySize, MachineIRBuilder MIRBuilder) const { 2598 assert(ArraySize == 1 && "Resource arrays are not implemented yet."); 2599 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 2600 MIRBuilder); 2601 } 2602 2603 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 2604 const SPIRVType *ResType, 2605 MachineInstr &I) const { 2606 // there was an allocation size parameter to the allocation instruction 2607 // that is not 1 2608 MachineBasicBlock &BB = *I.getParent(); 2609 return BuildMI(BB, I, I.getDebugLoc(), 2610 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 2611 .addDef(ResVReg) 2612 .addUse(GR.getSPIRVTypeID(ResType)) 2613 .addUse(I.getOperand(2).getReg()) 2614 .constrainAllUses(TII, TRI, RBI); 2615 } 2616 2617 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 2618 const SPIRVType *ResType, 2619 MachineInstr &I) const { 2620 // Change order of instructions if needed: all OpVariable instructions in a 2621 // function must be the first instructions in the first block 2622 MachineFunction *MF = I.getParent()->getParent(); 2623 MachineBasicBlock *MBB = &MF->front(); 2624 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 2625 bool IsHeader = false; 2626 unsigned Opcode; 2627 for (; It != E && It != I; ++It) { 2628 Opcode = It->getOpcode(); 2629 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 2630 IsHeader = true; 2631 } else if (IsHeader && 2632 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 2633 ++It; 2634 break; 2635 } 2636 } 2637 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 2638 .addDef(ResVReg) 2639 .addUse(GR.getSPIRVTypeID(ResType)) 2640 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 2641 .constrainAllUses(TII, TRI, RBI); 2642 } 2643 2644 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 2645 // InstructionSelector walks backwards through the instructions. We can use 2646 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 2647 // first, so can generate an OpBranchConditional here. If there is no 2648 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 2649 const MachineInstr *PrevI = I.getPrevNode(); 2650 MachineBasicBlock &MBB = *I.getParent(); 2651 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 2652 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2653 .addUse(PrevI->getOperand(0).getReg()) 2654 .addMBB(PrevI->getOperand(1).getMBB()) 2655 .addMBB(I.getOperand(0).getMBB()) 2656 .constrainAllUses(TII, TRI, RBI); 2657 } 2658 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 2659 .addMBB(I.getOperand(0).getMBB()) 2660 .constrainAllUses(TII, TRI, RBI); 2661 } 2662 2663 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 2664 // InstructionSelector walks backwards through the instructions. For an 2665 // explicit conditional branch with no fallthrough, we use both a G_BR and a 2666 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 2667 // generate the OpBranchConditional in selectBranch above. 2668 // 2669 // If an OpBranchConditional has been generated, we simply return, as the work 2670 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 2671 // implicit fallthrough to the next basic block, so we need to create an 2672 // OpBranchConditional with an explicit "false" argument pointing to the next 2673 // basic block that LLVM would fall through to. 2674 const MachineInstr *NextI = I.getNextNode(); 2675 // Check if this has already been successfully selected. 2676 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 2677 return true; 2678 // Must be relying on implicit block fallthrough, so generate an 2679 // OpBranchConditional with the "next" basic block as the "false" target. 2680 MachineBasicBlock &MBB = *I.getParent(); 2681 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 2682 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 2683 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2684 .addUse(I.getOperand(0).getReg()) 2685 .addMBB(I.getOperand(1).getMBB()) 2686 .addMBB(NextMBB) 2687 .constrainAllUses(TII, TRI, RBI); 2688 } 2689 2690 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 2691 const SPIRVType *ResType, 2692 MachineInstr &I) const { 2693 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 2694 .addDef(ResVReg) 2695 .addUse(GR.getSPIRVTypeID(ResType)); 2696 const unsigned NumOps = I.getNumOperands(); 2697 for (unsigned i = 1; i < NumOps; i += 2) { 2698 MIB.addUse(I.getOperand(i + 0).getReg()); 2699 MIB.addMBB(I.getOperand(i + 1).getMBB()); 2700 } 2701 return MIB.constrainAllUses(TII, TRI, RBI); 2702 } 2703 2704 bool SPIRVInstructionSelector::selectGlobalValue( 2705 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 2706 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 2707 MachineIRBuilder MIRBuilder(I); 2708 const GlobalValue *GV = I.getOperand(1).getGlobal(); 2709 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 2710 SPIRVType *PointerBaseType; 2711 if (GVType->isArrayTy()) { 2712 SPIRVType *ArrayElementType = 2713 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 2714 SPIRV::AccessQualifier::ReadWrite, false); 2715 PointerBaseType = GR.getOrCreateSPIRVArrayType( 2716 ArrayElementType, GVType->getArrayNumElements(), I, TII); 2717 } else { 2718 PointerBaseType = GR.getOrCreateSPIRVType( 2719 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 2720 } 2721 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 2722 PointerBaseType, I, TII, 2723 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 2724 2725 std::string GlobalIdent; 2726 if (!GV->hasName()) { 2727 unsigned &ID = UnnamedGlobalIDs[GV]; 2728 if (ID == 0) 2729 ID = UnnamedGlobalIDs.size(); 2730 GlobalIdent = "__unnamed_" + Twine(ID).str(); 2731 } else { 2732 GlobalIdent = GV->getGlobalIdentifier(); 2733 } 2734 2735 // Behaviour of functions as operands depends on availability of the 2736 // corresponding extension (SPV_INTEL_function_pointers): 2737 // - If there is an extension to operate with functions as operands: 2738 // We create a proper constant operand and evaluate a correct type for a 2739 // function pointer. 2740 // - Without the required extension: 2741 // We have functions as operands in tests with blocks of instruction e.g. in 2742 // transcoding/global_block.ll. These operands are not used and should be 2743 // substituted by zero constants. Their type is expected to be always 2744 // OpTypePointer Function %uchar. 2745 if (isa<Function>(GV)) { 2746 const Constant *ConstVal = GV; 2747 MachineBasicBlock &BB = *I.getParent(); 2748 Register NewReg = GR.find(ConstVal, GR.CurMF); 2749 if (!NewReg.isValid()) { 2750 Register NewReg = ResVReg; 2751 GR.add(ConstVal, GR.CurMF, NewReg); 2752 const Function *GVFun = 2753 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 2754 ? dyn_cast<Function>(GV) 2755 : nullptr; 2756 if (GVFun) { 2757 // References to a function via function pointers generate virtual 2758 // registers without a definition. We will resolve it later, during 2759 // module analysis stage. 2760 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2761 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2762 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 2763 MachineInstrBuilder MB = 2764 BuildMI(BB, I, I.getDebugLoc(), 2765 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 2766 .addDef(NewReg) 2767 .addUse(GR.getSPIRVTypeID(ResType)) 2768 .addUse(FuncVReg); 2769 // mapping the function pointer to the used Function 2770 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 2771 return MB.constrainAllUses(TII, TRI, RBI); 2772 } 2773 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2774 .addDef(NewReg) 2775 .addUse(GR.getSPIRVTypeID(ResType)) 2776 .constrainAllUses(TII, TRI, RBI); 2777 } 2778 assert(NewReg != ResVReg); 2779 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2780 .addDef(ResVReg) 2781 .addUse(NewReg) 2782 .constrainAllUses(TII, TRI, RBI); 2783 } 2784 auto GlobalVar = cast<GlobalVariable>(GV); 2785 assert(GlobalVar->getName() != "llvm.global.annotations"); 2786 2787 bool HasInit = GlobalVar->hasInitializer() && 2788 !isa<UndefValue>(GlobalVar->getInitializer()); 2789 // Skip empty declaration for GVs with initilaizers till we get the decl with 2790 // passed initializer. 2791 if (HasInit && !Init) 2792 return true; 2793 2794 unsigned AddrSpace = GV->getAddressSpace(); 2795 SPIRV::StorageClass::StorageClass Storage = 2796 addressSpaceToStorageClass(AddrSpace, STI); 2797 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 2798 Storage != SPIRV::StorageClass::Function; 2799 SPIRV::LinkageType::LinkageType LnkType = 2800 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 2801 ? SPIRV::LinkageType::Import 2802 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 2803 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 2804 ? SPIRV::LinkageType::LinkOnceODR 2805 : SPIRV::LinkageType::Export); 2806 2807 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 2808 Storage, Init, GlobalVar->isConstant(), 2809 HasLnkTy, LnkType, MIRBuilder, true); 2810 return Reg.isValid(); 2811 } 2812 2813 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 2814 const SPIRVType *ResType, 2815 MachineInstr &I) const { 2816 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 2817 return selectExtInst(ResVReg, ResType, I, CL::log10); 2818 } 2819 2820 // There is no log10 instruction in the GLSL Extended Instruction set, so it 2821 // is implemented as: 2822 // log10(x) = log2(x) * (1 / log2(10)) 2823 // = log2(x) * 0.30103 2824 2825 MachineIRBuilder MIRBuilder(I); 2826 MachineBasicBlock &BB = *I.getParent(); 2827 2828 // Build log2(x). 2829 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2830 bool Result = 2831 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2832 .addDef(VarReg) 2833 .addUse(GR.getSPIRVTypeID(ResType)) 2834 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2835 .addImm(GL::Log2) 2836 .add(I.getOperand(1)) 2837 .constrainAllUses(TII, TRI, RBI); 2838 2839 // Build 0.30103. 2840 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 2841 ResType->getOpcode() == SPIRV::OpTypeFloat); 2842 // TODO: Add matrix implementation once supported by the HLSL frontend. 2843 const SPIRVType *SpirvScalarType = 2844 ResType->getOpcode() == SPIRV::OpTypeVector 2845 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 2846 : ResType; 2847 Register ScaleReg = 2848 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 2849 2850 // Multiply log2(x) by 0.30103 to get log10(x) result. 2851 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 2852 ? SPIRV::OpVectorTimesScalar 2853 : SPIRV::OpFMulS; 2854 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2855 .addDef(ResVReg) 2856 .addUse(GR.getSPIRVTypeID(ResType)) 2857 .addUse(VarReg) 2858 .addUse(ScaleReg) 2859 .constrainAllUses(TII, TRI, RBI); 2860 2861 return Result; 2862 } 2863 2864 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 2865 const SPIRVType *ResType, 2866 MachineInstr &I) const { 2867 // DX intrinsic: @llvm.dx.thread.id(i32) 2868 // ID Name Description 2869 // 93 ThreadId reads the thread ID 2870 2871 MachineIRBuilder MIRBuilder(I); 2872 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 2873 const SPIRVType *Vec3Ty = 2874 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 2875 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 2876 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 2877 2878 // Create new register for GlobalInvocationID builtin variable. 2879 Register NewRegister = 2880 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 2881 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 2882 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 2883 2884 // Build GlobalInvocationID global variable with the necessary decorations. 2885 Register Variable = GR.buildGlobalVariable( 2886 NewRegister, PtrType, 2887 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 2888 SPIRV::StorageClass::Input, nullptr, true, true, 2889 SPIRV::LinkageType::Import, MIRBuilder, false); 2890 2891 // Create new register for loading value. 2892 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2893 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2894 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 2895 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 2896 2897 // Load v3uint value from the global variable. 2898 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2899 .addDef(LoadedRegister) 2900 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 2901 .addUse(Variable); 2902 2903 // Get Thread ID index. Expecting operand is a constant immediate value, 2904 // wrapped in a type assignment. 2905 assert(I.getOperand(2).isReg()); 2906 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 2907 2908 // Extract the thread ID from the loaded vector value. 2909 MachineBasicBlock &BB = *I.getParent(); 2910 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2911 .addDef(ResVReg) 2912 .addUse(GR.getSPIRVTypeID(ResType)) 2913 .addUse(LoadedRegister) 2914 .addImm(ThreadId); 2915 return MIB.constrainAllUses(TII, TRI, RBI); 2916 } 2917 2918 namespace llvm { 2919 InstructionSelector * 2920 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 2921 const SPIRVSubtarget &Subtarget, 2922 const RegisterBankInfo &RBI) { 2923 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 2924 } 2925 } // namespace llvm 2926