1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/Register.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/IR/IntrinsicsSPIRV.h" 34 #include "llvm/Support/Debug.h" 35 36 #define DEBUG_TYPE "spirv-isel" 37 38 using namespace llvm; 39 namespace CL = SPIRV::OpenCLExtInst; 40 namespace GL = SPIRV::GLSLExtInst; 41 42 using ExtInstList = 43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 44 45 namespace { 46 47 #define GET_GLOBALISEL_PREDICATE_BITSET 48 #include "SPIRVGenGlobalISel.inc" 49 #undef GET_GLOBALISEL_PREDICATE_BITSET 50 51 class SPIRVInstructionSelector : public InstructionSelector { 52 const SPIRVSubtarget &STI; 53 const SPIRVInstrInfo &TII; 54 const SPIRVRegisterInfo &TRI; 55 const RegisterBankInfo &RBI; 56 SPIRVGlobalRegistry &GR; 57 MachineRegisterInfo *MRI; 58 MachineFunction *HasVRegsReset = nullptr; 59 60 /// We need to keep track of the number we give to anonymous global values to 61 /// generate the same name every time when this is needed. 62 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 63 64 public: 65 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 66 const SPIRVSubtarget &ST, 67 const RegisterBankInfo &RBI); 68 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 69 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 70 BlockFrequencyInfo *BFI) override; 71 // Common selection code. Instruction-specific selection occurs in spvSelect. 72 bool select(MachineInstr &I) override; 73 static const char *getName() { return DEBUG_TYPE; } 74 75 #define GET_GLOBALISEL_PREDICATES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_PREDICATES_DECL 78 79 #define GET_GLOBALISEL_TEMPORARIES_DECL 80 #include "SPIRVGenGlobalISel.inc" 81 #undef GET_GLOBALISEL_TEMPORARIES_DECL 82 83 private: 84 void resetVRegsType(MachineFunction &MF); 85 86 // tblgen-erated 'select' implementation, used as the initial selector for 87 // the patterns that don't require complex C++. 88 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 89 90 // All instruction-specific selection that didn't happen in "select()". 91 // Is basically a large Switch/Case delegating to all other select method. 92 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I) const; 94 95 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 96 const MachineInstr *Init = nullptr) const; 97 98 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I, Register SrcReg, 100 unsigned Opcode) const; 101 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 102 unsigned Opcode) const; 103 104 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 105 MachineInstr &I) const; 106 107 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 108 MachineInstr &I) const; 109 bool selectStore(MachineInstr &I) const; 110 111 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I) const; 113 bool selectStackRestore(MachineInstr &I) const; 114 115 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 116 117 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 118 MachineInstr &I, unsigned NewOpcode, 119 unsigned NegateOpcode = 0) const; 120 121 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 bool selectFence(MachineInstr &I) const; 125 126 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 129 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 130 MachineInstr &I, unsigned OpType) const; 131 132 bool selectAll(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I) const; 134 135 bool selectAny(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I) const; 137 138 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 139 MachineInstr &I) const; 140 141 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 142 MachineInstr &I) const; 143 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 144 MachineInstr &I) const; 145 146 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 147 unsigned comparisonOpcode, MachineInstr &I) const; 148 bool selectCross(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectSign(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 158 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 162 MachineInstr &I, unsigned Opcode) const; 163 164 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 168 int OpIdx) const; 169 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 170 int OpIdx) const; 171 172 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 173 MachineInstr &I) const; 174 175 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 176 bool IsSigned) const; 177 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 178 bool IsSigned, unsigned Opcode) const; 179 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 180 bool IsSigned) const; 181 182 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I) const; 184 185 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 186 const SPIRVType *intTy, const SPIRVType *boolTy) const; 187 188 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 191 MachineInstr &I) const; 192 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 195 MachineInstr &I) const; 196 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 199 MachineInstr &I) const; 200 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 201 MachineInstr &I) const; 202 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I) const; 204 205 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I) const; 207 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 208 MachineInstr &I) const; 209 210 bool selectBranch(MachineInstr &I) const; 211 bool selectBranchCond(MachineInstr &I) const; 212 213 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 214 MachineInstr &I) const; 215 216 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 218 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I, CL::OpenCLExtInst CLInst, 220 GL::GLSLExtInst GLInst) const; 221 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I, const ExtInstList &ExtInsts) const; 223 224 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 227 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I) const; 229 230 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 233 bool selectUnmergeValues(MachineInstr &I) const; 234 235 // Utilities 236 Register buildI32Constant(uint32_t Val, MachineInstr &I, 237 const SPIRVType *ResType = nullptr) const; 238 239 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 240 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 241 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 242 MachineInstr &I) const; 243 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 244 245 bool wrapIntoSpecConstantOp(MachineInstr &I, 246 SmallVector<Register> &CompositeArgs) const; 247 248 Register getUcharPtrTypeReg(MachineInstr &I, 249 SPIRV::StorageClass::StorageClass SC) const; 250 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 251 Register Src, Register DestType, 252 uint32_t Opcode) const; 253 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 254 SPIRVType *SrcPtrTy) const; 255 }; 256 257 } // end anonymous namespace 258 259 #define GET_GLOBALISEL_IMPL 260 #include "SPIRVGenGlobalISel.inc" 261 #undef GET_GLOBALISEL_IMPL 262 263 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 264 const SPIRVSubtarget &ST, 265 const RegisterBankInfo &RBI) 266 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 267 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 268 #define GET_GLOBALISEL_PREDICATES_INIT 269 #include "SPIRVGenGlobalISel.inc" 270 #undef GET_GLOBALISEL_PREDICATES_INIT 271 #define GET_GLOBALISEL_TEMPORARIES_INIT 272 #include "SPIRVGenGlobalISel.inc" 273 #undef GET_GLOBALISEL_TEMPORARIES_INIT 274 { 275 } 276 277 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 278 CodeGenCoverage *CoverageInfo, 279 ProfileSummaryInfo *PSI, 280 BlockFrequencyInfo *BFI) { 281 MRI = &MF.getRegInfo(); 282 GR.setCurrentFunc(MF); 283 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 284 } 285 286 // Ensure that register classes correspond to pattern matching rules. 287 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 288 if (HasVRegsReset == &MF) 289 return; 290 HasVRegsReset = &MF; 291 292 MachineRegisterInfo &MRI = MF.getRegInfo(); 293 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 294 Register Reg = Register::index2VirtReg(I); 295 LLT RegType = MRI.getType(Reg); 296 if (RegType.isScalar()) 297 MRI.setType(Reg, LLT::scalar(64)); 298 else if (RegType.isPointer()) 299 MRI.setType(Reg, LLT::pointer(0, 64)); 300 else if (RegType.isVector()) 301 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 302 } 303 for (const auto &MBB : MF) { 304 for (const auto &MI : MBB) { 305 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 306 continue; 307 Register DstReg = MI.getOperand(0).getReg(); 308 LLT DstType = MRI.getType(DstReg); 309 Register SrcReg = MI.getOperand(1).getReg(); 310 LLT SrcType = MRI.getType(SrcReg); 311 if (DstType != SrcType) 312 MRI.setType(DstReg, MRI.getType(SrcReg)); 313 314 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 315 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 316 if (DstRC != SrcRC && SrcRC) 317 MRI.setRegClass(DstReg, SrcRC); 318 } 319 } 320 } 321 322 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 323 324 // Defined in SPIRVLegalizerInfo.cpp. 325 extern bool isTypeFoldingSupported(unsigned Opcode); 326 327 bool SPIRVInstructionSelector::select(MachineInstr &I) { 328 resetVRegsType(*I.getParent()->getParent()); 329 330 assert(I.getParent() && "Instruction should be in a basic block!"); 331 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 332 333 Register Opcode = I.getOpcode(); 334 // If it's not a GMIR instruction, we've selected it already. 335 if (!isPreISelGenericOpcode(Opcode)) { 336 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 337 Register DstReg = I.getOperand(0).getReg(); 338 Register SrcReg = I.getOperand(1).getReg(); 339 auto *Def = MRI->getVRegDef(SrcReg); 340 if (isTypeFoldingSupported(Def->getOpcode())) { 341 bool Res = selectImpl(I, *CoverageInfo); 342 LLVM_DEBUG({ 343 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 344 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 345 I.print(dbgs()); 346 } 347 }); 348 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 349 if (Res) 350 return Res; 351 } 352 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 353 MRI->replaceRegWith(SrcReg, DstReg); 354 I.removeFromParent(); 355 return true; 356 } else if (I.getNumDefs() == 1) { 357 // Make all vregs 64 bits (for SPIR-V IDs). 358 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 359 } 360 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 361 } 362 363 if (I.getNumOperands() != I.getNumExplicitOperands()) { 364 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 365 return false; 366 } 367 368 // Common code for getting return reg+type, and removing selected instr 369 // from parent occurs here. Instr-specific selection happens in spvSelect(). 370 bool HasDefs = I.getNumDefs() > 0; 371 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 372 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 373 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 374 if (spvSelect(ResVReg, ResType, I)) { 375 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 376 for (unsigned i = 0; i < I.getNumDefs(); ++i) 377 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 378 I.removeFromParent(); 379 return true; 380 } 381 return false; 382 } 383 384 static bool mayApplyGenericSelection(unsigned Opcode) { 385 switch (Opcode) { 386 case TargetOpcode::G_CONSTANT: 387 return false; 388 case TargetOpcode::G_SADDO: 389 case TargetOpcode::G_SSUBO: 390 return true; 391 } 392 return isTypeFoldingSupported(Opcode); 393 } 394 395 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 396 const SPIRVType *ResType, 397 MachineInstr &I) const { 398 const unsigned Opcode = I.getOpcode(); 399 if (mayApplyGenericSelection(Opcode)) 400 return selectImpl(I, *CoverageInfo); 401 switch (Opcode) { 402 case TargetOpcode::G_CONSTANT: 403 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 404 I); 405 case TargetOpcode::G_GLOBAL_VALUE: 406 return selectGlobalValue(ResVReg, I); 407 case TargetOpcode::G_IMPLICIT_DEF: 408 return selectOpUndef(ResVReg, ResType, I); 409 case TargetOpcode::G_FREEZE: 410 return selectFreeze(ResVReg, ResType, I); 411 412 case TargetOpcode::G_INTRINSIC: 413 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 414 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 415 return selectIntrinsic(ResVReg, ResType, I); 416 case TargetOpcode::G_BITREVERSE: 417 return selectBitreverse(ResVReg, ResType, I); 418 419 case TargetOpcode::G_BUILD_VECTOR: 420 return selectBuildVector(ResVReg, ResType, I); 421 case TargetOpcode::G_SPLAT_VECTOR: 422 return selectSplatVector(ResVReg, ResType, I); 423 424 case TargetOpcode::G_SHUFFLE_VECTOR: { 425 MachineBasicBlock &BB = *I.getParent(); 426 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 427 .addDef(ResVReg) 428 .addUse(GR.getSPIRVTypeID(ResType)) 429 .addUse(I.getOperand(1).getReg()) 430 .addUse(I.getOperand(2).getReg()); 431 for (auto V : I.getOperand(3).getShuffleMask()) 432 MIB.addImm(V); 433 return MIB.constrainAllUses(TII, TRI, RBI); 434 } 435 case TargetOpcode::G_MEMMOVE: 436 case TargetOpcode::G_MEMCPY: 437 case TargetOpcode::G_MEMSET: 438 return selectMemOperation(ResVReg, I); 439 440 case TargetOpcode::G_ICMP: 441 return selectICmp(ResVReg, ResType, I); 442 case TargetOpcode::G_FCMP: 443 return selectFCmp(ResVReg, ResType, I); 444 445 case TargetOpcode::G_FRAME_INDEX: 446 return selectFrameIndex(ResVReg, ResType, I); 447 448 case TargetOpcode::G_LOAD: 449 return selectLoad(ResVReg, ResType, I); 450 case TargetOpcode::G_STORE: 451 return selectStore(I); 452 453 case TargetOpcode::G_BR: 454 return selectBranch(I); 455 case TargetOpcode::G_BRCOND: 456 return selectBranchCond(I); 457 458 case TargetOpcode::G_PHI: 459 return selectPhi(ResVReg, ResType, I); 460 461 case TargetOpcode::G_FPTOSI: 462 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 463 case TargetOpcode::G_FPTOUI: 464 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 465 466 case TargetOpcode::G_SITOFP: 467 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 468 case TargetOpcode::G_UITOFP: 469 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 470 471 case TargetOpcode::G_CTPOP: 472 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 473 case TargetOpcode::G_SMIN: 474 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 475 case TargetOpcode::G_UMIN: 476 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 477 478 case TargetOpcode::G_SMAX: 479 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 480 case TargetOpcode::G_UMAX: 481 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 482 483 case TargetOpcode::G_FMA: 484 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 485 486 case TargetOpcode::G_FPOW: 487 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 488 case TargetOpcode::G_FPOWI: 489 return selectExtInst(ResVReg, ResType, I, CL::pown); 490 491 case TargetOpcode::G_FEXP: 492 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 493 case TargetOpcode::G_FEXP2: 494 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 495 496 case TargetOpcode::G_FLOG: 497 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 498 case TargetOpcode::G_FLOG2: 499 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 500 case TargetOpcode::G_FLOG10: 501 return selectLog10(ResVReg, ResType, I); 502 503 case TargetOpcode::G_FABS: 504 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 505 case TargetOpcode::G_ABS: 506 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 507 508 case TargetOpcode::G_FMINNUM: 509 case TargetOpcode::G_FMINIMUM: 510 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 511 case TargetOpcode::G_FMAXNUM: 512 case TargetOpcode::G_FMAXIMUM: 513 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 514 515 case TargetOpcode::G_FCOPYSIGN: 516 return selectExtInst(ResVReg, ResType, I, CL::copysign); 517 518 case TargetOpcode::G_FCEIL: 519 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 520 case TargetOpcode::G_FFLOOR: 521 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 522 523 case TargetOpcode::G_FCOS: 524 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 525 case TargetOpcode::G_FSIN: 526 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 527 case TargetOpcode::G_FTAN: 528 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 529 case TargetOpcode::G_FACOS: 530 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 531 case TargetOpcode::G_FASIN: 532 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 533 case TargetOpcode::G_FATAN: 534 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 535 case TargetOpcode::G_FATAN2: 536 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 537 case TargetOpcode::G_FCOSH: 538 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 539 case TargetOpcode::G_FSINH: 540 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 541 case TargetOpcode::G_FTANH: 542 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 543 544 case TargetOpcode::G_FSQRT: 545 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 546 547 case TargetOpcode::G_CTTZ: 548 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 549 return selectExtInst(ResVReg, ResType, I, CL::ctz); 550 case TargetOpcode::G_CTLZ: 551 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 552 return selectExtInst(ResVReg, ResType, I, CL::clz); 553 554 case TargetOpcode::G_INTRINSIC_ROUND: 555 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 556 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 557 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 558 case TargetOpcode::G_INTRINSIC_TRUNC: 559 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 560 case TargetOpcode::G_FRINT: 561 case TargetOpcode::G_FNEARBYINT: 562 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 563 564 case TargetOpcode::G_SMULH: 565 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 566 case TargetOpcode::G_UMULH: 567 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 568 569 case TargetOpcode::G_SADDSAT: 570 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 571 case TargetOpcode::G_UADDSAT: 572 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 573 case TargetOpcode::G_SSUBSAT: 574 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 575 case TargetOpcode::G_USUBSAT: 576 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 577 578 case TargetOpcode::G_UADDO: 579 return selectOverflowArith(ResVReg, ResType, I, 580 ResType->getOpcode() == SPIRV::OpTypeVector 581 ? SPIRV::OpIAddCarryV 582 : SPIRV::OpIAddCarryS); 583 case TargetOpcode::G_USUBO: 584 return selectOverflowArith(ResVReg, ResType, I, 585 ResType->getOpcode() == SPIRV::OpTypeVector 586 ? SPIRV::OpISubBorrowV 587 : SPIRV::OpISubBorrowS); 588 case TargetOpcode::G_UMULO: 589 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 590 case TargetOpcode::G_SMULO: 591 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 592 593 case TargetOpcode::G_SEXT: 594 return selectExt(ResVReg, ResType, I, true); 595 case TargetOpcode::G_ANYEXT: 596 case TargetOpcode::G_ZEXT: 597 return selectExt(ResVReg, ResType, I, false); 598 case TargetOpcode::G_TRUNC: 599 return selectTrunc(ResVReg, ResType, I); 600 case TargetOpcode::G_FPTRUNC: 601 case TargetOpcode::G_FPEXT: 602 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 603 604 case TargetOpcode::G_PTRTOINT: 605 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 606 case TargetOpcode::G_INTTOPTR: 607 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 608 case TargetOpcode::G_BITCAST: 609 return selectBitcast(ResVReg, ResType, I); 610 case TargetOpcode::G_ADDRSPACE_CAST: 611 return selectAddrSpaceCast(ResVReg, ResType, I); 612 case TargetOpcode::G_PTR_ADD: { 613 // Currently, we get G_PTR_ADD only applied to global variables. 614 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 615 Register GV = I.getOperand(1).getReg(); 616 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 617 (void)II; 618 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 619 (*II).getOpcode() == TargetOpcode::COPY || 620 (*II).getOpcode() == SPIRV::OpVariable) && 621 isImm(I.getOperand(2), MRI)); 622 // It may be the initialization of a global variable. 623 bool IsGVInit = false; 624 for (MachineRegisterInfo::use_instr_iterator 625 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 626 UseEnd = MRI->use_instr_end(); 627 UseIt != UseEnd; UseIt = std::next(UseIt)) { 628 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 629 (*UseIt).getOpcode() == SPIRV::OpVariable) { 630 IsGVInit = true; 631 break; 632 } 633 } 634 MachineBasicBlock &BB = *I.getParent(); 635 if (!IsGVInit) { 636 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 637 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 638 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 639 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 640 // Build a new virtual register that is associated with the required 641 // data type. 642 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 643 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 644 // Having a correctly typed base we are ready to build the actually 645 // required GEP. It may not be a constant though, because all Operands 646 // of OpSpecConstantOp is to originate from other const instructions, 647 // and only the AccessChain named opcodes accept a global OpVariable 648 // instruction. We can't use an AccessChain opcode because of the type 649 // mismatch between result and base types. 650 if (!GR.isBitcastCompatible(ResType, GVType)) 651 report_fatal_error( 652 "incompatible result and operand types in a bitcast"); 653 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 654 MachineInstrBuilder MIB = 655 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 656 .addDef(NewVReg) 657 .addUse(ResTypeReg) 658 .addUse(GV); 659 return MIB.constrainAllUses(TII, TRI, RBI) && 660 BuildMI(BB, I, I.getDebugLoc(), 661 TII.get(STI.isVulkanEnv() 662 ? SPIRV::OpInBoundsAccessChain 663 : SPIRV::OpInBoundsPtrAccessChain)) 664 .addDef(ResVReg) 665 .addUse(ResTypeReg) 666 .addUse(NewVReg) 667 .addUse(I.getOperand(2).getReg()) 668 .constrainAllUses(TII, TRI, RBI); 669 } else { 670 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 671 .addDef(ResVReg) 672 .addUse(GR.getSPIRVTypeID(ResType)) 673 .addImm( 674 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 675 .addUse(GV) 676 .addUse(I.getOperand(2).getReg()) 677 .constrainAllUses(TII, TRI, RBI); 678 } 679 } 680 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 681 // initialize a global variable with a constant expression (e.g., the test 682 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 683 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 684 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 685 .addDef(ResVReg) 686 .addUse(GR.getSPIRVTypeID(ResType)) 687 .addImm(static_cast<uint32_t>( 688 SPIRV::Opcode::InBoundsPtrAccessChain)) 689 .addUse(GV) 690 .addUse(Idx) 691 .addUse(I.getOperand(2).getReg()); 692 return MIB.constrainAllUses(TII, TRI, RBI); 693 } 694 695 case TargetOpcode::G_ATOMICRMW_OR: 696 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 697 case TargetOpcode::G_ATOMICRMW_ADD: 698 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 699 case TargetOpcode::G_ATOMICRMW_AND: 700 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 701 case TargetOpcode::G_ATOMICRMW_MAX: 702 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 703 case TargetOpcode::G_ATOMICRMW_MIN: 704 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 705 case TargetOpcode::G_ATOMICRMW_SUB: 706 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 707 case TargetOpcode::G_ATOMICRMW_XOR: 708 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 709 case TargetOpcode::G_ATOMICRMW_UMAX: 710 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 711 case TargetOpcode::G_ATOMICRMW_UMIN: 712 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 713 case TargetOpcode::G_ATOMICRMW_XCHG: 714 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 715 case TargetOpcode::G_ATOMIC_CMPXCHG: 716 return selectAtomicCmpXchg(ResVReg, ResType, I); 717 718 case TargetOpcode::G_ATOMICRMW_FADD: 719 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 720 case TargetOpcode::G_ATOMICRMW_FSUB: 721 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 722 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 723 SPIRV::OpFNegate); 724 case TargetOpcode::G_ATOMICRMW_FMIN: 725 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 726 case TargetOpcode::G_ATOMICRMW_FMAX: 727 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 728 729 case TargetOpcode::G_FENCE: 730 return selectFence(I); 731 732 case TargetOpcode::G_STACKSAVE: 733 return selectStackSave(ResVReg, ResType, I); 734 case TargetOpcode::G_STACKRESTORE: 735 return selectStackRestore(I); 736 737 case TargetOpcode::G_UNMERGE_VALUES: 738 return selectUnmergeValues(I); 739 740 // Discard gen opcodes for intrinsics which we do not expect to actually 741 // represent code after lowering or intrinsics which are not implemented but 742 // should not crash when found in a customer's LLVM IR input. 743 case TargetOpcode::G_TRAP: 744 case TargetOpcode::G_DEBUGTRAP: 745 case TargetOpcode::G_UBSANTRAP: 746 case TargetOpcode::DBG_LABEL: 747 return true; 748 749 default: 750 return false; 751 } 752 } 753 754 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 755 const SPIRVType *ResType, 756 MachineInstr &I, 757 CL::OpenCLExtInst CLInst) const { 758 return selectExtInst(ResVReg, ResType, I, 759 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 760 } 761 762 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 763 const SPIRVType *ResType, 764 MachineInstr &I, 765 CL::OpenCLExtInst CLInst, 766 GL::GLSLExtInst GLInst) const { 767 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 768 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 769 return selectExtInst(ResVReg, ResType, I, ExtInsts); 770 } 771 772 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 773 const SPIRVType *ResType, 774 MachineInstr &I, 775 const ExtInstList &Insts) const { 776 777 for (const auto &Ex : Insts) { 778 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 779 uint32_t Opcode = Ex.second; 780 if (STI.canUseExtInstSet(Set)) { 781 MachineBasicBlock &BB = *I.getParent(); 782 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 783 .addDef(ResVReg) 784 .addUse(GR.getSPIRVTypeID(ResType)) 785 .addImm(static_cast<uint32_t>(Set)) 786 .addImm(Opcode); 787 const unsigned NumOps = I.getNumOperands(); 788 unsigned Index = 1; 789 if (Index < NumOps && 790 I.getOperand(Index).getType() == 791 MachineOperand::MachineOperandType::MO_IntrinsicID) 792 Index = 2; 793 for (; Index < NumOps; ++Index) 794 MIB.add(I.getOperand(Index)); 795 return MIB.constrainAllUses(TII, TRI, RBI); 796 } 797 } 798 return false; 799 } 800 801 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 802 const SPIRVType *ResType, 803 MachineInstr &I, 804 Register SrcReg, 805 unsigned Opcode) const { 806 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 807 .addDef(ResVReg) 808 .addUse(GR.getSPIRVTypeID(ResType)) 809 .addUse(SrcReg) 810 .constrainAllUses(TII, TRI, RBI); 811 } 812 813 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 814 const SPIRVType *ResType, 815 MachineInstr &I, 816 unsigned Opcode) const { 817 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 818 Register SrcReg = I.getOperand(1).getReg(); 819 bool IsGV = false; 820 for (MachineRegisterInfo::def_instr_iterator DefIt = 821 MRI->def_instr_begin(SrcReg); 822 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 823 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 824 IsGV = true; 825 break; 826 } 827 } 828 if (IsGV) { 829 uint32_t SpecOpcode = 0; 830 switch (Opcode) { 831 case SPIRV::OpConvertPtrToU: 832 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 833 break; 834 case SPIRV::OpConvertUToPtr: 835 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 836 break; 837 } 838 if (SpecOpcode) 839 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 840 TII.get(SPIRV::OpSpecConstantOp)) 841 .addDef(ResVReg) 842 .addUse(GR.getSPIRVTypeID(ResType)) 843 .addImm(SpecOpcode) 844 .addUse(SrcReg) 845 .constrainAllUses(TII, TRI, RBI); 846 } 847 } 848 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 849 Opcode); 850 } 851 852 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 853 const SPIRVType *ResType, 854 MachineInstr &I) const { 855 Register OpReg = I.getOperand(1).getReg(); 856 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 857 if (!GR.isBitcastCompatible(ResType, OpType)) 858 report_fatal_error("incompatible result and operand types in a bitcast"); 859 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 860 } 861 862 static void addMemoryOperands(MachineMemOperand *MemOp, 863 MachineInstrBuilder &MIB) { 864 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 865 if (MemOp->isVolatile()) 866 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 867 if (MemOp->isNonTemporal()) 868 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 869 if (MemOp->getAlign().value()) 870 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 871 872 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 873 MIB.addImm(SpvMemOp); 874 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 875 MIB.addImm(MemOp->getAlign().value()); 876 } 877 } 878 879 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 880 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 881 if (Flags & MachineMemOperand::Flags::MOVolatile) 882 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 883 if (Flags & MachineMemOperand::Flags::MONonTemporal) 884 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 885 886 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 887 MIB.addImm(SpvMemOp); 888 } 889 890 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 891 const SPIRVType *ResType, 892 MachineInstr &I) const { 893 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 894 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 895 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 896 .addDef(ResVReg) 897 .addUse(GR.getSPIRVTypeID(ResType)) 898 .addUse(Ptr); 899 if (!I.getNumMemOperands()) { 900 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 901 I.getOpcode() == 902 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 903 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 904 } else { 905 addMemoryOperands(*I.memoperands_begin(), MIB); 906 } 907 return MIB.constrainAllUses(TII, TRI, RBI); 908 } 909 910 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 911 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 912 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 913 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 914 MachineBasicBlock &BB = *I.getParent(); 915 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 916 .addUse(Ptr) 917 .addUse(StoreVal); 918 if (!I.getNumMemOperands()) { 919 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 920 I.getOpcode() == 921 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 922 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 923 } else { 924 addMemoryOperands(*I.memoperands_begin(), MIB); 925 } 926 return MIB.constrainAllUses(TII, TRI, RBI); 927 } 928 929 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 930 const SPIRVType *ResType, 931 MachineInstr &I) const { 932 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 933 report_fatal_error( 934 "llvm.stacksave intrinsic: this instruction requires the following " 935 "SPIR-V extension: SPV_INTEL_variable_length_array", 936 false); 937 MachineBasicBlock &BB = *I.getParent(); 938 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 939 .addDef(ResVReg) 940 .addUse(GR.getSPIRVTypeID(ResType)) 941 .constrainAllUses(TII, TRI, RBI); 942 } 943 944 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 945 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 946 report_fatal_error( 947 "llvm.stackrestore intrinsic: this instruction requires the following " 948 "SPIR-V extension: SPV_INTEL_variable_length_array", 949 false); 950 if (!I.getOperand(0).isReg()) 951 return false; 952 MachineBasicBlock &BB = *I.getParent(); 953 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 954 .addUse(I.getOperand(0).getReg()) 955 .constrainAllUses(TII, TRI, RBI); 956 } 957 958 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 959 MachineInstr &I) const { 960 MachineBasicBlock &BB = *I.getParent(); 961 Register SrcReg = I.getOperand(1).getReg(); 962 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 963 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 964 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 965 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 966 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 967 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 968 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 969 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 970 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 971 // TODO: check if we have such GV, add init, use buildGlobalVariable. 972 Function &CurFunction = GR.CurMF->getFunction(); 973 Type *LLVMArrTy = 974 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 975 // Module takes ownership of the global var. 976 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 977 true, GlobalValue::InternalLinkage, 978 Constant::getNullValue(LLVMArrTy)); 979 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 980 GR.add(GV, GR.CurMF, VarReg); 981 982 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 983 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 984 .addDef(VarReg) 985 .addUse(GR.getSPIRVTypeID(VarTy)) 986 .addImm(SPIRV::StorageClass::UniformConstant) 987 .addUse(Const) 988 .constrainAllUses(TII, TRI, RBI); 989 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 990 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 991 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 992 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 993 } 994 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 995 .addUse(I.getOperand(0).getReg()) 996 .addUse(SrcReg) 997 .addUse(I.getOperand(2).getReg()); 998 if (I.getNumMemOperands()) 999 addMemoryOperands(*I.memoperands_begin(), MIB); 1000 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1001 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1002 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1003 .addUse(MIB->getOperand(0).getReg()); 1004 return Result; 1005 } 1006 1007 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1008 const SPIRVType *ResType, 1009 MachineInstr &I, 1010 unsigned NewOpcode, 1011 unsigned NegateOpcode) const { 1012 assert(I.hasOneMemOperand()); 1013 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1014 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1015 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1016 Register ScopeReg = buildI32Constant(Scope, I); 1017 1018 Register Ptr = I.getOperand(1).getReg(); 1019 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1020 // auto ScSem = 1021 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1022 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1023 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1024 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 1025 1026 bool Result = false; 1027 Register ValueReg = I.getOperand(2).getReg(); 1028 if (NegateOpcode != 0) { 1029 // Translation with negative value operand is requested 1030 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1031 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1032 ValueReg = TmpReg; 1033 } 1034 1035 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1036 .addDef(ResVReg) 1037 .addUse(GR.getSPIRVTypeID(ResType)) 1038 .addUse(Ptr) 1039 .addUse(ScopeReg) 1040 .addUse(MemSemReg) 1041 .addUse(ValueReg) 1042 .constrainAllUses(TII, TRI, RBI); 1043 return Result; 1044 } 1045 1046 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1047 unsigned ArgI = I.getNumOperands() - 1; 1048 Register SrcReg = 1049 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1050 SPIRVType *DefType = 1051 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1052 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1053 report_fatal_error( 1054 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1055 1056 SPIRVType *ScalarType = 1057 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1058 MachineBasicBlock &BB = *I.getParent(); 1059 bool Res = false; 1060 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1061 Register ResVReg = I.getOperand(i).getReg(); 1062 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1063 if (!ResType) { 1064 // There was no "assign type" actions, let's fix this now 1065 ResType = ScalarType; 1066 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1067 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1068 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1069 } 1070 auto MIB = 1071 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1072 .addDef(ResVReg) 1073 .addUse(GR.getSPIRVTypeID(ResType)) 1074 .addUse(SrcReg) 1075 .addImm(static_cast<int64_t>(i)); 1076 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1077 } 1078 return Res; 1079 } 1080 1081 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1082 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1083 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1084 Register MemSemReg = buildI32Constant(MemSem, I); 1085 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1086 uint32_t Scope = static_cast<uint32_t>( 1087 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1088 Register ScopeReg = buildI32Constant(Scope, I); 1089 MachineBasicBlock &BB = *I.getParent(); 1090 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1091 .addUse(ScopeReg) 1092 .addUse(MemSemReg) 1093 .constrainAllUses(TII, TRI, RBI); 1094 } 1095 1096 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1097 const SPIRVType *ResType, 1098 MachineInstr &I, 1099 unsigned Opcode) const { 1100 Type *ResTy = nullptr; 1101 StringRef ResName; 1102 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1103 report_fatal_error( 1104 "Not enough info to select the arithmetic with overflow instruction"); 1105 if (!ResTy || !ResTy->isStructTy()) 1106 report_fatal_error("Expect struct type result for the arithmetic " 1107 "with overflow instruction"); 1108 // "Result Type must be from OpTypeStruct. The struct must have two members, 1109 // and the two members must be the same type." 1110 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1111 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1112 // Build SPIR-V types and constant(s) if needed. 1113 MachineIRBuilder MIRBuilder(I); 1114 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1115 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1116 assert(I.getNumDefs() > 1 && "Not enought operands"); 1117 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1118 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1119 if (N > 1) 1120 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1121 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1122 Register ZeroReg = buildZerosVal(ResType, I); 1123 // A new virtual register to store the result struct. 1124 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1125 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1126 // Build the result name if needed. 1127 if (ResName.size() > 0) 1128 buildOpName(StructVReg, ResName, MIRBuilder); 1129 // Build the arithmetic with overflow instruction. 1130 MachineBasicBlock &BB = *I.getParent(); 1131 auto MIB = 1132 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1133 .addDef(StructVReg) 1134 .addUse(GR.getSPIRVTypeID(StructType)); 1135 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1136 MIB.addUse(I.getOperand(i).getReg()); 1137 bool Status = MIB.constrainAllUses(TII, TRI, RBI); 1138 // Build instructions to extract fields of the instruction's result. 1139 // A new virtual register to store the higher part of the result struct. 1140 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1141 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1142 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1143 auto MIB = 1144 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1145 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1146 .addUse(GR.getSPIRVTypeID(ResType)) 1147 .addUse(StructVReg) 1148 .addImm(i); 1149 Status &= MIB.constrainAllUses(TII, TRI, RBI); 1150 } 1151 // Build boolean value from the higher part. 1152 Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1153 .addDef(I.getOperand(1).getReg()) 1154 .addUse(BoolTypeReg) 1155 .addUse(HigherVReg) 1156 .addUse(ZeroReg) 1157 .constrainAllUses(TII, TRI, RBI); 1158 return Status; 1159 } 1160 1161 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1162 const SPIRVType *ResType, 1163 MachineInstr &I) const { 1164 Register ScopeReg; 1165 Register MemSemEqReg; 1166 Register MemSemNeqReg; 1167 Register Ptr = I.getOperand(2).getReg(); 1168 if (!isa<GIntrinsic>(I)) { 1169 assert(I.hasOneMemOperand()); 1170 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1171 unsigned Scope = static_cast<uint32_t>(getMemScope( 1172 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1173 ScopeReg = buildI32Constant(Scope, I); 1174 1175 unsigned ScSem = static_cast<uint32_t>( 1176 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1177 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1178 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1179 MemSemEqReg = buildI32Constant(MemSemEq, I); 1180 AtomicOrdering FO = MemOp->getFailureOrdering(); 1181 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1182 MemSemNeqReg = 1183 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 1184 } else { 1185 ScopeReg = I.getOperand(5).getReg(); 1186 MemSemEqReg = I.getOperand(6).getReg(); 1187 MemSemNeqReg = I.getOperand(7).getReg(); 1188 } 1189 1190 Register Cmp = I.getOperand(3).getReg(); 1191 Register Val = I.getOperand(4).getReg(); 1192 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1193 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1194 const DebugLoc &DL = I.getDebugLoc(); 1195 bool Result = 1196 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1197 .addDef(ACmpRes) 1198 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1199 .addUse(Ptr) 1200 .addUse(ScopeReg) 1201 .addUse(MemSemEqReg) 1202 .addUse(MemSemNeqReg) 1203 .addUse(Val) 1204 .addUse(Cmp) 1205 .constrainAllUses(TII, TRI, RBI); 1206 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1207 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1208 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1209 .addDef(CmpSuccReg) 1210 .addUse(GR.getSPIRVTypeID(BoolTy)) 1211 .addUse(ACmpRes) 1212 .addUse(Cmp) 1213 .constrainAllUses(TII, TRI, RBI); 1214 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1215 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1216 .addDef(TmpReg) 1217 .addUse(GR.getSPIRVTypeID(ResType)) 1218 .addUse(ACmpRes) 1219 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1220 .addImm(0) 1221 .constrainAllUses(TII, TRI, RBI); 1222 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1223 .addDef(ResVReg) 1224 .addUse(GR.getSPIRVTypeID(ResType)) 1225 .addUse(CmpSuccReg) 1226 .addUse(TmpReg) 1227 .addImm(1) 1228 .constrainAllUses(TII, TRI, RBI); 1229 return Result; 1230 } 1231 1232 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1233 switch (SC) { 1234 case SPIRV::StorageClass::Workgroup: 1235 case SPIRV::StorageClass::CrossWorkgroup: 1236 case SPIRV::StorageClass::Function: 1237 return true; 1238 default: 1239 return false; 1240 } 1241 } 1242 1243 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1244 switch (SC) { 1245 case SPIRV::StorageClass::DeviceOnlyINTEL: 1246 case SPIRV::StorageClass::HostOnlyINTEL: 1247 return true; 1248 default: 1249 return false; 1250 } 1251 } 1252 1253 // Returns true ResVReg is referred only from global vars and OpName's. 1254 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1255 bool IsGRef = false; 1256 bool IsAllowedRefs = 1257 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1258 [&IsGRef](auto const &It) { 1259 unsigned Opcode = It.getOpcode(); 1260 if (Opcode == SPIRV::OpConstantComposite || 1261 Opcode == SPIRV::OpVariable || 1262 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1263 return IsGRef = true; 1264 return Opcode == SPIRV::OpName; 1265 }); 1266 return IsAllowedRefs && IsGRef; 1267 } 1268 1269 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1270 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1271 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1272 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1273 } 1274 1275 MachineInstrBuilder 1276 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1277 Register Src, Register DestType, 1278 uint32_t Opcode) const { 1279 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1280 TII.get(SPIRV::OpSpecConstantOp)) 1281 .addDef(Dest) 1282 .addUse(DestType) 1283 .addImm(Opcode) 1284 .addUse(Src); 1285 } 1286 1287 MachineInstrBuilder 1288 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1289 SPIRVType *SrcPtrTy) const { 1290 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1291 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1292 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1293 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1294 SPIRV::StorageClass::Generic), 1295 GR.getPointerSize())); 1296 MachineFunction *MF = I.getParent()->getParent(); 1297 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1298 MachineInstrBuilder MIB = buildSpecConstantOp( 1299 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1300 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1301 GR.add(MIB.getInstr(), MF, Tmp); 1302 return MIB; 1303 } 1304 1305 // In SPIR-V address space casting can only happen to and from the Generic 1306 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1307 // pointers to and from Generic pointers. As such, we can convert e.g. from 1308 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1309 // other combinations can only be done by a bitcast, and are probably not safe. 1310 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1311 const SPIRVType *ResType, 1312 MachineInstr &I) const { 1313 MachineBasicBlock &BB = *I.getParent(); 1314 const DebugLoc &DL = I.getDebugLoc(); 1315 1316 Register SrcPtr = I.getOperand(1).getReg(); 1317 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1318 1319 // don't generate a cast for a null that may be represented by OpTypeInt 1320 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1321 ResType->getOpcode() != SPIRV::OpTypePointer) 1322 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1323 .addDef(ResVReg) 1324 .addUse(SrcPtr) 1325 .constrainAllUses(TII, TRI, RBI); 1326 1327 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1328 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1329 1330 if (isASCastInGVar(MRI, ResVReg)) { 1331 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1332 // are expressed by OpSpecConstantOp with an Opcode. 1333 // TODO: maybe insert a check whether the Kernel capability was declared and 1334 // so PtrCastToGeneric/GenericCastToPtr are available. 1335 unsigned SpecOpcode = 1336 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1337 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1338 : (SrcSC == SPIRV::StorageClass::Generic && 1339 isGenericCastablePtr(DstSC) 1340 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1341 : 0); 1342 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1343 // correct value of ResType and use general i8* instead. Maybe this should 1344 // be addressed in the emit-intrinsic step to infer a correct 1345 // OpConstantComposite type. 1346 if (SpecOpcode) { 1347 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1348 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1349 .constrainAllUses(TII, TRI, RBI); 1350 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1351 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1352 return MIB.constrainAllUses(TII, TRI, RBI) && 1353 buildSpecConstantOp( 1354 I, ResVReg, MIB->getOperand(0).getReg(), 1355 getUcharPtrTypeReg(I, DstSC), 1356 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1357 .constrainAllUses(TII, TRI, RBI); 1358 } 1359 } 1360 1361 // don't generate a cast between identical storage classes 1362 if (SrcSC == DstSC) 1363 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1364 .addDef(ResVReg) 1365 .addUse(SrcPtr) 1366 .constrainAllUses(TII, TRI, RBI); 1367 1368 // Casting from an eligible pointer to Generic. 1369 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1370 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1371 // Casting from Generic to an eligible pointer. 1372 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1373 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1374 // Casting between 2 eligible pointers using Generic as an intermediary. 1375 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1376 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1377 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1378 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1379 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1380 .addDef(Tmp) 1381 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1382 .addUse(SrcPtr) 1383 .constrainAllUses(TII, TRI, RBI); 1384 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1385 .addDef(ResVReg) 1386 .addUse(GR.getSPIRVTypeID(ResType)) 1387 .addUse(Tmp) 1388 .constrainAllUses(TII, TRI, RBI); 1389 } 1390 1391 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1392 // be applied 1393 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1394 return selectUnOp(ResVReg, ResType, I, 1395 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1396 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1397 return selectUnOp(ResVReg, ResType, I, 1398 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1399 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1400 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1401 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1402 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1403 1404 // Bitcast for pointers requires that the address spaces must match 1405 return false; 1406 } 1407 1408 static unsigned getFCmpOpcode(unsigned PredNum) { 1409 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1410 switch (Pred) { 1411 case CmpInst::FCMP_OEQ: 1412 return SPIRV::OpFOrdEqual; 1413 case CmpInst::FCMP_OGE: 1414 return SPIRV::OpFOrdGreaterThanEqual; 1415 case CmpInst::FCMP_OGT: 1416 return SPIRV::OpFOrdGreaterThan; 1417 case CmpInst::FCMP_OLE: 1418 return SPIRV::OpFOrdLessThanEqual; 1419 case CmpInst::FCMP_OLT: 1420 return SPIRV::OpFOrdLessThan; 1421 case CmpInst::FCMP_ONE: 1422 return SPIRV::OpFOrdNotEqual; 1423 case CmpInst::FCMP_ORD: 1424 return SPIRV::OpOrdered; 1425 case CmpInst::FCMP_UEQ: 1426 return SPIRV::OpFUnordEqual; 1427 case CmpInst::FCMP_UGE: 1428 return SPIRV::OpFUnordGreaterThanEqual; 1429 case CmpInst::FCMP_UGT: 1430 return SPIRV::OpFUnordGreaterThan; 1431 case CmpInst::FCMP_ULE: 1432 return SPIRV::OpFUnordLessThanEqual; 1433 case CmpInst::FCMP_ULT: 1434 return SPIRV::OpFUnordLessThan; 1435 case CmpInst::FCMP_UNE: 1436 return SPIRV::OpFUnordNotEqual; 1437 case CmpInst::FCMP_UNO: 1438 return SPIRV::OpUnordered; 1439 default: 1440 llvm_unreachable("Unknown predicate type for FCmp"); 1441 } 1442 } 1443 1444 static unsigned getICmpOpcode(unsigned PredNum) { 1445 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1446 switch (Pred) { 1447 case CmpInst::ICMP_EQ: 1448 return SPIRV::OpIEqual; 1449 case CmpInst::ICMP_NE: 1450 return SPIRV::OpINotEqual; 1451 case CmpInst::ICMP_SGE: 1452 return SPIRV::OpSGreaterThanEqual; 1453 case CmpInst::ICMP_SGT: 1454 return SPIRV::OpSGreaterThan; 1455 case CmpInst::ICMP_SLE: 1456 return SPIRV::OpSLessThanEqual; 1457 case CmpInst::ICMP_SLT: 1458 return SPIRV::OpSLessThan; 1459 case CmpInst::ICMP_UGE: 1460 return SPIRV::OpUGreaterThanEqual; 1461 case CmpInst::ICMP_UGT: 1462 return SPIRV::OpUGreaterThan; 1463 case CmpInst::ICMP_ULE: 1464 return SPIRV::OpULessThanEqual; 1465 case CmpInst::ICMP_ULT: 1466 return SPIRV::OpULessThan; 1467 default: 1468 llvm_unreachable("Unknown predicate type for ICmp"); 1469 } 1470 } 1471 1472 static unsigned getPtrCmpOpcode(unsigned Pred) { 1473 switch (static_cast<CmpInst::Predicate>(Pred)) { 1474 case CmpInst::ICMP_EQ: 1475 return SPIRV::OpPtrEqual; 1476 case CmpInst::ICMP_NE: 1477 return SPIRV::OpPtrNotEqual; 1478 default: 1479 llvm_unreachable("Unknown predicate type for pointer comparison"); 1480 } 1481 } 1482 1483 // Return the logical operation, or abort if none exists. 1484 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1485 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1486 switch (Pred) { 1487 case CmpInst::ICMP_EQ: 1488 return SPIRV::OpLogicalEqual; 1489 case CmpInst::ICMP_NE: 1490 return SPIRV::OpLogicalNotEqual; 1491 default: 1492 llvm_unreachable("Unknown predicate type for Bool comparison"); 1493 } 1494 } 1495 1496 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1497 if (!LLVMFloatTy) 1498 return APFloat::getZero(APFloat::IEEEsingle()); 1499 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1500 case Type::HalfTyID: 1501 return APFloat::getZero(APFloat::IEEEhalf()); 1502 default: 1503 case Type::FloatTyID: 1504 return APFloat::getZero(APFloat::IEEEsingle()); 1505 case Type::DoubleTyID: 1506 return APFloat::getZero(APFloat::IEEEdouble()); 1507 } 1508 } 1509 1510 static APFloat getOneFP(const Type *LLVMFloatTy) { 1511 if (!LLVMFloatTy) 1512 return APFloat::getOne(APFloat::IEEEsingle()); 1513 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1514 case Type::HalfTyID: 1515 return APFloat::getOne(APFloat::IEEEhalf()); 1516 default: 1517 case Type::FloatTyID: 1518 return APFloat::getOne(APFloat::IEEEsingle()); 1519 case Type::DoubleTyID: 1520 return APFloat::getOne(APFloat::IEEEdouble()); 1521 } 1522 } 1523 1524 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1525 const SPIRVType *ResType, 1526 MachineInstr &I, 1527 unsigned OpAnyOrAll) const { 1528 assert(I.getNumOperands() == 3); 1529 assert(I.getOperand(2).isReg()); 1530 MachineBasicBlock &BB = *I.getParent(); 1531 Register InputRegister = I.getOperand(2).getReg(); 1532 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1533 1534 if (!InputType) 1535 report_fatal_error("Input Type could not be determined."); 1536 1537 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1538 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1539 if (IsBoolTy && !IsVectorTy) { 1540 assert(ResVReg == I.getOperand(0).getReg()); 1541 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1542 TII.get(TargetOpcode::COPY)) 1543 .addDef(ResVReg) 1544 .addUse(InputRegister) 1545 .constrainAllUses(TII, TRI, RBI); 1546 } 1547 1548 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1549 unsigned SpirvNotEqualId = 1550 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1551 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1552 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1553 Register NotEqualReg = ResVReg; 1554 1555 if (IsVectorTy) { 1556 NotEqualReg = IsBoolTy ? InputRegister 1557 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1558 const unsigned NumElts = InputType->getOperand(2).getImm(); 1559 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1560 } 1561 1562 if (!IsBoolTy) { 1563 Register ConstZeroReg = 1564 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1565 1566 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1567 .addDef(NotEqualReg) 1568 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1569 .addUse(InputRegister) 1570 .addUse(ConstZeroReg) 1571 .constrainAllUses(TII, TRI, RBI); 1572 } 1573 1574 if (!IsVectorTy) 1575 return true; 1576 1577 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1578 .addDef(ResVReg) 1579 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1580 .addUse(NotEqualReg) 1581 .constrainAllUses(TII, TRI, RBI); 1582 } 1583 1584 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1585 const SPIRVType *ResType, 1586 MachineInstr &I) const { 1587 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1588 } 1589 1590 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1591 const SPIRVType *ResType, 1592 MachineInstr &I) const { 1593 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1594 } 1595 1596 // Select the OpDot instruction for the given float dot 1597 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1598 const SPIRVType *ResType, 1599 MachineInstr &I) const { 1600 assert(I.getNumOperands() == 4); 1601 assert(I.getOperand(2).isReg()); 1602 assert(I.getOperand(3).isReg()); 1603 1604 [[maybe_unused]] SPIRVType *VecType = 1605 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1606 1607 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1608 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1609 "dot product requires a vector of at least 2 components"); 1610 1611 [[maybe_unused]] SPIRVType *EltType = 1612 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1613 1614 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1615 1616 MachineBasicBlock &BB = *I.getParent(); 1617 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1618 .addDef(ResVReg) 1619 .addUse(GR.getSPIRVTypeID(ResType)) 1620 .addUse(I.getOperand(2).getReg()) 1621 .addUse(I.getOperand(3).getReg()) 1622 .constrainAllUses(TII, TRI, RBI); 1623 } 1624 1625 // Since pre-1.6 SPIRV has no integer dot implementation, 1626 // expand by piecewise multiplying and adding the results 1627 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1628 const SPIRVType *ResType, 1629 MachineInstr &I) const { 1630 assert(I.getNumOperands() == 4); 1631 assert(I.getOperand(2).isReg()); 1632 assert(I.getOperand(3).isReg()); 1633 MachineBasicBlock &BB = *I.getParent(); 1634 1635 // Multiply the vectors, then sum the results 1636 Register Vec0 = I.getOperand(2).getReg(); 1637 Register Vec1 = I.getOperand(3).getReg(); 1638 Register TmpVec = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1639 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1640 1641 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1642 .addDef(TmpVec) 1643 .addUse(GR.getSPIRVTypeID(VecType)) 1644 .addUse(Vec0) 1645 .addUse(Vec1) 1646 .constrainAllUses(TII, TRI, RBI); 1647 1648 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1649 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1650 "dot product requires a vector of at least 2 components"); 1651 1652 Register Res = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1653 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1654 .addDef(Res) 1655 .addUse(GR.getSPIRVTypeID(ResType)) 1656 .addUse(TmpVec) 1657 .addImm(0) 1658 .constrainAllUses(TII, TRI, RBI); 1659 1660 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1661 Register Elt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1662 1663 Result |= 1664 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1665 .addDef(Elt) 1666 .addUse(GR.getSPIRVTypeID(ResType)) 1667 .addUse(TmpVec) 1668 .addImm(i) 1669 .constrainAllUses(TII, TRI, RBI); 1670 1671 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1672 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1673 : ResVReg; 1674 1675 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1676 .addDef(Sum) 1677 .addUse(GR.getSPIRVTypeID(ResType)) 1678 .addUse(Res) 1679 .addUse(Elt) 1680 .constrainAllUses(TII, TRI, RBI); 1681 Res = Sum; 1682 } 1683 1684 return Result; 1685 } 1686 1687 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1688 /// does not have a saturate builtin. 1689 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1690 const SPIRVType *ResType, 1691 MachineInstr &I) const { 1692 assert(I.getNumOperands() == 3); 1693 assert(I.getOperand(2).isReg()); 1694 MachineBasicBlock &BB = *I.getParent(); 1695 Register VZero = buildZerosValF(ResType, I); 1696 Register VOne = buildOnesValF(ResType, I); 1697 1698 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1699 .addDef(ResVReg) 1700 .addUse(GR.getSPIRVTypeID(ResType)) 1701 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1702 .addImm(GL::FClamp) 1703 .addUse(I.getOperand(2).getReg()) 1704 .addUse(VZero) 1705 .addUse(VOne) 1706 .constrainAllUses(TII, TRI, RBI); 1707 } 1708 1709 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1710 const SPIRVType *ResType, 1711 MachineInstr &I) const { 1712 assert(I.getNumOperands() == 3); 1713 assert(I.getOperand(2).isReg()); 1714 MachineBasicBlock &BB = *I.getParent(); 1715 Register InputRegister = I.getOperand(2).getReg(); 1716 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1717 auto &DL = I.getDebugLoc(); 1718 1719 if (!InputType) 1720 report_fatal_error("Input Type could not be determined."); 1721 1722 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1723 1724 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1725 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1726 1727 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1728 1729 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1730 Register SignReg = NeedsConversion 1731 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1732 : ResVReg; 1733 1734 bool Result = 1735 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1736 .addDef(SignReg) 1737 .addUse(GR.getSPIRVTypeID(InputType)) 1738 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1739 .addImm(SignOpcode) 1740 .addUse(InputRegister) 1741 .constrainAllUses(TII, TRI, RBI); 1742 1743 if (NeedsConversion) { 1744 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1745 Result |= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1746 .addDef(ResVReg) 1747 .addUse(GR.getSPIRVTypeID(ResType)) 1748 .addUse(SignReg) 1749 .constrainAllUses(TII, TRI, RBI); 1750 } 1751 1752 return Result; 1753 } 1754 1755 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1756 const SPIRVType *ResType, 1757 MachineInstr &I) const { 1758 MachineBasicBlock &BB = *I.getParent(); 1759 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1760 .addDef(ResVReg) 1761 .addUse(GR.getSPIRVTypeID(ResType)) 1762 .addUse(I.getOperand(1).getReg()) 1763 .constrainAllUses(TII, TRI, RBI); 1764 } 1765 1766 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1767 const SPIRVType *ResType, 1768 MachineInstr &I) const { 1769 // There is no way to implement `freeze` correctly without support on SPIR-V 1770 // standard side, but we may at least address a simple (static) case when 1771 // undef/poison value presence is obvious. The main benefit of even 1772 // incomplete `freeze` support is preventing of translation from crashing due 1773 // to lack of support on legalization and instruction selection steps. 1774 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1775 return false; 1776 Register OpReg = I.getOperand(1).getReg(); 1777 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1778 Register Reg; 1779 switch (Def->getOpcode()) { 1780 case SPIRV::ASSIGN_TYPE: 1781 if (MachineInstr *AssignToDef = 1782 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1783 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1784 Reg = Def->getOperand(2).getReg(); 1785 } 1786 break; 1787 case SPIRV::OpUndef: 1788 Reg = Def->getOperand(1).getReg(); 1789 break; 1790 } 1791 unsigned DestOpCode; 1792 if (Reg.isValid()) { 1793 DestOpCode = SPIRV::OpConstantNull; 1794 } else { 1795 DestOpCode = TargetOpcode::COPY; 1796 Reg = OpReg; 1797 } 1798 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1799 .addDef(I.getOperand(0).getReg()) 1800 .addUse(Reg) 1801 .constrainAllUses(TII, TRI, RBI); 1802 } 1803 return false; 1804 } 1805 1806 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1807 const SPIRVType *ResType) { 1808 Register OpReg = ResType->getOperand(2).getReg(); 1809 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1810 if (!OpDef) 1811 return 0; 1812 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1813 OpDef->getOperand(1).isReg()) { 1814 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1815 OpDef = RefDef; 1816 } 1817 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 1818 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 1819 : 0; 1820 return N; 1821 } 1822 1823 // Return true if the type represents a constant register 1824 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 1825 SmallPtrSet<SPIRVType *, 4> &Visited) { 1826 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1827 OpDef->getOperand(1).isReg()) { 1828 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1829 OpDef = RefDef; 1830 } 1831 1832 if (Visited.contains(OpDef)) 1833 return true; 1834 Visited.insert(OpDef); 1835 1836 unsigned Opcode = OpDef->getOpcode(); 1837 switch (Opcode) { 1838 case TargetOpcode::G_CONSTANT: 1839 case TargetOpcode::G_FCONSTANT: 1840 return true; 1841 case TargetOpcode::G_INTRINSIC: 1842 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 1843 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 1844 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 1845 Intrinsic::spv_const_composite; 1846 case TargetOpcode::G_BUILD_VECTOR: 1847 case TargetOpcode::G_SPLAT_VECTOR: { 1848 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 1849 i++) { 1850 SPIRVType *OpNestedDef = 1851 OpDef->getOperand(i).isReg() 1852 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 1853 : nullptr; 1854 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 1855 return false; 1856 } 1857 return true; 1858 } 1859 } 1860 return false; 1861 } 1862 1863 // Return true if the virtual register represents a constant 1864 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 1865 SmallPtrSet<SPIRVType *, 4> Visited; 1866 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 1867 return isConstReg(MRI, OpDef, Visited); 1868 return false; 1869 } 1870 1871 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 1872 const SPIRVType *ResType, 1873 MachineInstr &I) const { 1874 unsigned N = 0; 1875 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1876 N = GR.getScalarOrVectorComponentCount(ResType); 1877 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1878 N = getArrayComponentCount(MRI, ResType); 1879 else 1880 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 1881 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 1882 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 1883 1884 // check if we may construct a constant vector 1885 bool IsConst = true; 1886 for (unsigned i = I.getNumExplicitDefs(); 1887 i < I.getNumExplicitOperands() && IsConst; ++i) 1888 if (!isConstReg(MRI, I.getOperand(i).getReg())) 1889 IsConst = false; 1890 1891 if (!IsConst && N < 2) 1892 report_fatal_error( 1893 "There must be at least two constituent operands in a vector"); 1894 1895 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1896 TII.get(IsConst ? SPIRV::OpConstantComposite 1897 : SPIRV::OpCompositeConstruct)) 1898 .addDef(ResVReg) 1899 .addUse(GR.getSPIRVTypeID(ResType)); 1900 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 1901 MIB.addUse(I.getOperand(i).getReg()); 1902 return MIB.constrainAllUses(TII, TRI, RBI); 1903 } 1904 1905 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 1906 const SPIRVType *ResType, 1907 MachineInstr &I) const { 1908 unsigned N = 0; 1909 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1910 N = GR.getScalarOrVectorComponentCount(ResType); 1911 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1912 N = getArrayComponentCount(MRI, ResType); 1913 else 1914 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 1915 1916 unsigned OpIdx = I.getNumExplicitDefs(); 1917 if (!I.getOperand(OpIdx).isReg()) 1918 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 1919 1920 // check if we may construct a constant vector 1921 Register OpReg = I.getOperand(OpIdx).getReg(); 1922 bool IsConst = isConstReg(MRI, OpReg); 1923 1924 if (!IsConst && N < 2) 1925 report_fatal_error( 1926 "There must be at least two constituent operands in a vector"); 1927 1928 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1929 TII.get(IsConst ? SPIRV::OpConstantComposite 1930 : SPIRV::OpCompositeConstruct)) 1931 .addDef(ResVReg) 1932 .addUse(GR.getSPIRVTypeID(ResType)); 1933 for (unsigned i = 0; i < N; ++i) 1934 MIB.addUse(OpReg); 1935 return MIB.constrainAllUses(TII, TRI, RBI); 1936 } 1937 1938 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 1939 const SPIRVType *ResType, 1940 unsigned CmpOpc, 1941 MachineInstr &I) const { 1942 Register Cmp0 = I.getOperand(2).getReg(); 1943 Register Cmp1 = I.getOperand(3).getReg(); 1944 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1945 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1946 "CMP operands should have the same type"); 1947 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1948 .addDef(ResVReg) 1949 .addUse(GR.getSPIRVTypeID(ResType)) 1950 .addUse(Cmp0) 1951 .addUse(Cmp1) 1952 .constrainAllUses(TII, TRI, RBI); 1953 } 1954 1955 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1956 const SPIRVType *ResType, 1957 MachineInstr &I) const { 1958 auto Pred = I.getOperand(1).getPredicate(); 1959 unsigned CmpOpc; 1960 1961 Register CmpOperand = I.getOperand(2).getReg(); 1962 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1963 CmpOpc = getPtrCmpOpcode(Pred); 1964 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1965 CmpOpc = getBoolCmpOpcode(Pred); 1966 else 1967 CmpOpc = getICmpOpcode(Pred); 1968 return selectCmp(ResVReg, ResType, CmpOpc, I); 1969 } 1970 1971 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 1972 const MachineInstr &I, 1973 int OpIdx) const { 1974 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1975 "Expected G_FCONSTANT"); 1976 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1977 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1978 } 1979 1980 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1981 const MachineInstr &I, 1982 int OpIdx) const { 1983 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1984 "Expected G_CONSTANT"); 1985 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1986 } 1987 1988 Register 1989 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1990 const SPIRVType *ResType) const { 1991 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1992 const SPIRVType *SpvI32Ty = 1993 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1994 // Find a constant in DT or build a new one. 1995 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1996 Register NewReg = GR.find(ConstInt, GR.CurMF); 1997 if (!NewReg.isValid()) { 1998 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1999 GR.add(ConstInt, GR.CurMF, NewReg); 2000 MachineInstr *MI; 2001 MachineBasicBlock &BB = *I.getParent(); 2002 if (Val == 0) { 2003 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2004 .addDef(NewReg) 2005 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2006 } else { 2007 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2008 .addDef(NewReg) 2009 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2010 .addImm(APInt(32, Val).getZExtValue()); 2011 } 2012 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2013 } 2014 return NewReg; 2015 } 2016 2017 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2018 const SPIRVType *ResType, 2019 MachineInstr &I) const { 2020 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2021 return selectCmp(ResVReg, ResType, CmpOp, I); 2022 } 2023 2024 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2025 MachineInstr &I) const { 2026 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2027 bool ZeroAsNull = STI.isOpenCLEnv(); 2028 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2029 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2030 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2031 } 2032 2033 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2034 MachineInstr &I) const { 2035 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2036 bool ZeroAsNull = STI.isOpenCLEnv(); 2037 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2038 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2039 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2040 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2041 } 2042 2043 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2044 MachineInstr &I) const { 2045 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2046 bool ZeroAsNull = STI.isOpenCLEnv(); 2047 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2048 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2049 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2050 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2051 } 2052 2053 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2054 const SPIRVType *ResType, 2055 MachineInstr &I) const { 2056 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2057 APInt One = 2058 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2059 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2060 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2061 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2062 } 2063 2064 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2065 const SPIRVType *ResType, 2066 MachineInstr &I, 2067 bool IsSigned) const { 2068 // To extend a bool, we need to use OpSelect between constants. 2069 Register ZeroReg = buildZerosVal(ResType, I); 2070 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2071 bool IsScalarBool = 2072 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2073 unsigned Opcode = 2074 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2075 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2076 .addDef(ResVReg) 2077 .addUse(GR.getSPIRVTypeID(ResType)) 2078 .addUse(I.getOperand(1).getReg()) 2079 .addUse(OneReg) 2080 .addUse(ZeroReg) 2081 .constrainAllUses(TII, TRI, RBI); 2082 } 2083 2084 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2085 const SPIRVType *ResType, 2086 MachineInstr &I, bool IsSigned, 2087 unsigned Opcode) const { 2088 Register SrcReg = I.getOperand(1).getReg(); 2089 // We can convert bool value directly to float type without OpConvert*ToF, 2090 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2091 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2092 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2093 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2094 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2095 const unsigned NumElts = ResType->getOperand(2).getImm(); 2096 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2097 } 2098 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2099 selectSelect(SrcReg, TmpType, I, false); 2100 } 2101 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2102 } 2103 2104 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2105 const SPIRVType *ResType, 2106 MachineInstr &I, bool IsSigned) const { 2107 Register SrcReg = I.getOperand(1).getReg(); 2108 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2109 return selectSelect(ResVReg, ResType, I, IsSigned); 2110 2111 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2112 if (SrcType == ResType) { 2113 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2114 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2115 if (DstRC != SrcRC && SrcRC) 2116 MRI->setRegClass(ResVReg, SrcRC); 2117 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2118 TII.get(TargetOpcode::COPY)) 2119 .addDef(ResVReg) 2120 .addUse(SrcReg) 2121 .constrainAllUses(TII, TRI, RBI); 2122 } 2123 2124 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2125 return selectUnOp(ResVReg, ResType, I, Opcode); 2126 } 2127 2128 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2129 Register ResVReg, 2130 MachineInstr &I, 2131 const SPIRVType *IntTy, 2132 const SPIRVType *BoolTy) const { 2133 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2134 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2135 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2136 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2137 Register Zero = buildZerosVal(IntTy, I); 2138 Register One = buildOnesVal(false, IntTy, I); 2139 MachineBasicBlock &BB = *I.getParent(); 2140 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2141 .addDef(BitIntReg) 2142 .addUse(GR.getSPIRVTypeID(IntTy)) 2143 .addUse(IntReg) 2144 .addUse(One) 2145 .constrainAllUses(TII, TRI, RBI); 2146 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2147 .addDef(ResVReg) 2148 .addUse(GR.getSPIRVTypeID(BoolTy)) 2149 .addUse(BitIntReg) 2150 .addUse(Zero) 2151 .constrainAllUses(TII, TRI, RBI); 2152 } 2153 2154 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2155 const SPIRVType *ResType, 2156 MachineInstr &I) const { 2157 Register IntReg = I.getOperand(1).getReg(); 2158 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2159 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2160 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2161 if (ArgType == ResType) { 2162 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2163 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2164 if (DstRC != SrcRC && SrcRC) 2165 MRI->setRegClass(ResVReg, SrcRC); 2166 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2167 TII.get(TargetOpcode::COPY)) 2168 .addDef(ResVReg) 2169 .addUse(IntReg) 2170 .constrainAllUses(TII, TRI, RBI); 2171 } 2172 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2173 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2174 return selectUnOp(ResVReg, ResType, I, Opcode); 2175 } 2176 2177 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2178 const SPIRVType *ResType, 2179 const APInt &Imm, 2180 MachineInstr &I) const { 2181 unsigned TyOpcode = ResType->getOpcode(); 2182 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2183 MachineBasicBlock &BB = *I.getParent(); 2184 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2185 Imm.isZero()) 2186 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2187 .addDef(ResVReg) 2188 .addUse(GR.getSPIRVTypeID(ResType)) 2189 .constrainAllUses(TII, TRI, RBI); 2190 if (TyOpcode == SPIRV::OpTypeInt) { 2191 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2192 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2193 if (Reg == ResVReg) 2194 return true; 2195 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2196 .addDef(ResVReg) 2197 .addUse(Reg) 2198 .constrainAllUses(TII, TRI, RBI); 2199 } 2200 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2201 .addDef(ResVReg) 2202 .addUse(GR.getSPIRVTypeID(ResType)); 2203 // <=32-bit integers should be caught by the sdag pattern. 2204 assert(Imm.getBitWidth() > 32); 2205 addNumImm(Imm, MIB); 2206 return MIB.constrainAllUses(TII, TRI, RBI); 2207 } 2208 2209 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2210 const SPIRVType *ResType, 2211 MachineInstr &I) const { 2212 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2213 .addDef(ResVReg) 2214 .addUse(GR.getSPIRVTypeID(ResType)) 2215 .constrainAllUses(TII, TRI, RBI); 2216 } 2217 2218 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2219 assert(MO.isReg()); 2220 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2221 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2222 assert(TypeInst->getOperand(1).isReg()); 2223 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2224 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2225 } 2226 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2227 } 2228 2229 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2230 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2231 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2232 return TypeInst->getOperand(2).getImm(); 2233 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2234 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2235 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2236 } 2237 2238 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2239 const SPIRVType *ResType, 2240 MachineInstr &I) const { 2241 MachineBasicBlock &BB = *I.getParent(); 2242 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2243 .addDef(ResVReg) 2244 .addUse(GR.getSPIRVTypeID(ResType)) 2245 // object to insert 2246 .addUse(I.getOperand(3).getReg()) 2247 // composite to insert into 2248 .addUse(I.getOperand(2).getReg()); 2249 for (unsigned i = 4; i < I.getNumOperands(); i++) 2250 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2251 return MIB.constrainAllUses(TII, TRI, RBI); 2252 } 2253 2254 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2255 const SPIRVType *ResType, 2256 MachineInstr &I) const { 2257 MachineBasicBlock &BB = *I.getParent(); 2258 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2259 .addDef(ResVReg) 2260 .addUse(GR.getSPIRVTypeID(ResType)) 2261 .addUse(I.getOperand(2).getReg()); 2262 for (unsigned i = 3; i < I.getNumOperands(); i++) 2263 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2264 return MIB.constrainAllUses(TII, TRI, RBI); 2265 } 2266 2267 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2268 const SPIRVType *ResType, 2269 MachineInstr &I) const { 2270 if (isImm(I.getOperand(4), MRI)) 2271 return selectInsertVal(ResVReg, ResType, I); 2272 MachineBasicBlock &BB = *I.getParent(); 2273 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2274 .addDef(ResVReg) 2275 .addUse(GR.getSPIRVTypeID(ResType)) 2276 .addUse(I.getOperand(2).getReg()) 2277 .addUse(I.getOperand(3).getReg()) 2278 .addUse(I.getOperand(4).getReg()) 2279 .constrainAllUses(TII, TRI, RBI); 2280 } 2281 2282 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2283 const SPIRVType *ResType, 2284 MachineInstr &I) const { 2285 if (isImm(I.getOperand(3), MRI)) 2286 return selectExtractVal(ResVReg, ResType, I); 2287 MachineBasicBlock &BB = *I.getParent(); 2288 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2289 .addDef(ResVReg) 2290 .addUse(GR.getSPIRVTypeID(ResType)) 2291 .addUse(I.getOperand(2).getReg()) 2292 .addUse(I.getOperand(3).getReg()) 2293 .constrainAllUses(TII, TRI, RBI); 2294 } 2295 2296 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2297 const SPIRVType *ResType, 2298 MachineInstr &I) const { 2299 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2300 2301 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2302 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2303 // we have to use Op[InBounds]AccessChain. 2304 const unsigned Opcode = STI.isVulkanEnv() 2305 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2306 : SPIRV::OpAccessChain) 2307 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2308 : SPIRV::OpPtrAccessChain); 2309 2310 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2311 .addDef(ResVReg) 2312 .addUse(GR.getSPIRVTypeID(ResType)) 2313 // Object to get a pointer to. 2314 .addUse(I.getOperand(3).getReg()); 2315 // Adding indices. 2316 const unsigned StartingIndex = 2317 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2318 ? 5 2319 : 4; 2320 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2321 Res.addUse(I.getOperand(i).getReg()); 2322 return Res.constrainAllUses(TII, TRI, RBI); 2323 } 2324 2325 // Maybe wrap a value into OpSpecConstantOp 2326 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2327 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2328 bool Result = true; 2329 unsigned Lim = I.getNumExplicitOperands(); 2330 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2331 Register OpReg = I.getOperand(i).getReg(); 2332 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2333 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2334 SmallPtrSet<SPIRVType *, 4> Visited; 2335 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2336 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2337 GR.isAggregateType(OpType)) { 2338 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2339 // by selectAddrSpaceCast() 2340 CompositeArgs.push_back(OpReg); 2341 continue; 2342 } 2343 MachineFunction *MF = I.getMF(); 2344 Register WrapReg = GR.find(OpDefine, MF); 2345 if (WrapReg.isValid()) { 2346 CompositeArgs.push_back(WrapReg); 2347 continue; 2348 } 2349 // Create a new register for the wrapper 2350 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2351 GR.add(OpDefine, MF, WrapReg); 2352 CompositeArgs.push_back(WrapReg); 2353 // Decorate the wrapper register and generate a new instruction 2354 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2355 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2356 MachineBasicBlock &BB = *I.getParent(); 2357 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2358 .addDef(WrapReg) 2359 .addUse(GR.getSPIRVTypeID(OpType)) 2360 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2361 .addUse(OpReg) 2362 .constrainAllUses(TII, TRI, RBI); 2363 if (!Result) 2364 break; 2365 } 2366 return Result; 2367 } 2368 2369 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2370 const SPIRVType *ResType, 2371 MachineInstr &I) const { 2372 MachineBasicBlock &BB = *I.getParent(); 2373 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2374 switch (IID) { 2375 case Intrinsic::spv_load: 2376 return selectLoad(ResVReg, ResType, I); 2377 case Intrinsic::spv_store: 2378 return selectStore(I); 2379 case Intrinsic::spv_extractv: 2380 return selectExtractVal(ResVReg, ResType, I); 2381 case Intrinsic::spv_insertv: 2382 return selectInsertVal(ResVReg, ResType, I); 2383 case Intrinsic::spv_extractelt: 2384 return selectExtractElt(ResVReg, ResType, I); 2385 case Intrinsic::spv_insertelt: 2386 return selectInsertElt(ResVReg, ResType, I); 2387 case Intrinsic::spv_gep: 2388 return selectGEP(ResVReg, ResType, I); 2389 case Intrinsic::spv_unref_global: 2390 case Intrinsic::spv_init_global: { 2391 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2392 MachineInstr *Init = I.getNumExplicitOperands() > 2 2393 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2394 : nullptr; 2395 assert(MI); 2396 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2397 } 2398 case Intrinsic::spv_undef: { 2399 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2400 .addDef(ResVReg) 2401 .addUse(GR.getSPIRVTypeID(ResType)); 2402 return MIB.constrainAllUses(TII, TRI, RBI); 2403 } 2404 case Intrinsic::spv_const_composite: { 2405 // If no values are attached, the composite is null constant. 2406 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2407 // Select a proper instruction. 2408 unsigned Opcode = SPIRV::OpConstantNull; 2409 SmallVector<Register> CompositeArgs; 2410 if (!IsNull) { 2411 Opcode = SPIRV::OpConstantComposite; 2412 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2413 return false; 2414 } 2415 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2416 .addDef(ResVReg) 2417 .addUse(GR.getSPIRVTypeID(ResType)); 2418 // skip type MD node we already used when generated assign.type for this 2419 if (!IsNull) { 2420 for (Register OpReg : CompositeArgs) 2421 MIB.addUse(OpReg); 2422 } 2423 return MIB.constrainAllUses(TII, TRI, RBI); 2424 } 2425 case Intrinsic::spv_assign_name: { 2426 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2427 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2428 for (unsigned i = I.getNumExplicitDefs() + 2; 2429 i < I.getNumExplicitOperands(); ++i) { 2430 MIB.addImm(I.getOperand(i).getImm()); 2431 } 2432 return MIB.constrainAllUses(TII, TRI, RBI); 2433 } 2434 case Intrinsic::spv_switch: { 2435 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2436 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2437 if (I.getOperand(i).isReg()) 2438 MIB.addReg(I.getOperand(i).getReg()); 2439 else if (I.getOperand(i).isCImm()) 2440 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2441 else if (I.getOperand(i).isMBB()) 2442 MIB.addMBB(I.getOperand(i).getMBB()); 2443 else 2444 llvm_unreachable("Unexpected OpSwitch operand"); 2445 } 2446 return MIB.constrainAllUses(TII, TRI, RBI); 2447 } 2448 case Intrinsic::spv_loop_merge: 2449 case Intrinsic::spv_selection_merge: { 2450 const auto Opcode = IID == Intrinsic::spv_selection_merge 2451 ? SPIRV::OpSelectionMerge 2452 : SPIRV::OpLoopMerge; 2453 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2454 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2455 assert(I.getOperand(i).isMBB()); 2456 MIB.addMBB(I.getOperand(i).getMBB()); 2457 } 2458 MIB.addImm(SPIRV::SelectionControl::None); 2459 return MIB.constrainAllUses(TII, TRI, RBI); 2460 } 2461 case Intrinsic::spv_cmpxchg: 2462 return selectAtomicCmpXchg(ResVReg, ResType, I); 2463 case Intrinsic::spv_unreachable: 2464 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 2465 break; 2466 case Intrinsic::spv_alloca: 2467 return selectFrameIndex(ResVReg, ResType, I); 2468 case Intrinsic::spv_alloca_array: 2469 return selectAllocaArray(ResVReg, ResType, I); 2470 case Intrinsic::spv_assume: 2471 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2472 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2473 .addUse(I.getOperand(1).getReg()); 2474 break; 2475 case Intrinsic::spv_expect: 2476 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2477 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2478 .addDef(ResVReg) 2479 .addUse(GR.getSPIRVTypeID(ResType)) 2480 .addUse(I.getOperand(2).getReg()) 2481 .addUse(I.getOperand(3).getReg()); 2482 break; 2483 case Intrinsic::arithmetic_fence: 2484 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2485 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT)) 2486 .addDef(ResVReg) 2487 .addUse(GR.getSPIRVTypeID(ResType)) 2488 .addUse(I.getOperand(2).getReg()); 2489 else 2490 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 2491 .addUse(I.getOperand(2).getReg()); 2492 break; 2493 case Intrinsic::spv_thread_id: 2494 return selectSpvThreadId(ResVReg, ResType, I); 2495 case Intrinsic::spv_fdot: 2496 return selectFloatDot(ResVReg, ResType, I); 2497 case Intrinsic::spv_udot: 2498 case Intrinsic::spv_sdot: 2499 return selectIntegerDot(ResVReg, ResType, I); 2500 case Intrinsic::spv_all: 2501 return selectAll(ResVReg, ResType, I); 2502 case Intrinsic::spv_any: 2503 return selectAny(ResVReg, ResType, I); 2504 case Intrinsic::spv_cross: 2505 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2506 case Intrinsic::spv_lerp: 2507 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2508 case Intrinsic::spv_length: 2509 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2510 case Intrinsic::spv_frac: 2511 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2512 case Intrinsic::spv_normalize: 2513 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2514 case Intrinsic::spv_rsqrt: 2515 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2516 case Intrinsic::spv_sign: 2517 return selectSign(ResVReg, ResType, I); 2518 case Intrinsic::spv_lifetime_start: 2519 case Intrinsic::spv_lifetime_end: { 2520 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2521 : SPIRV::OpLifetimeStop; 2522 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2523 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2524 if (Size == -1) 2525 Size = 0; 2526 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 2527 } break; 2528 case Intrinsic::spv_saturate: 2529 return selectSaturate(ResVReg, ResType, I); 2530 case Intrinsic::spv_wave_is_first_lane: { 2531 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2532 return BuildMI(BB, I, I.getDebugLoc(), 2533 TII.get(SPIRV::OpGroupNonUniformElect)) 2534 .addDef(ResVReg) 2535 .addUse(GR.getSPIRVTypeID(ResType)) 2536 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2537 } 2538 case Intrinsic::spv_step: 2539 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2540 case Intrinsic::spv_radians: 2541 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2542 // Discard intrinsics which we do not expect to actually represent code after 2543 // lowering or intrinsics which are not implemented but should not crash when 2544 // found in a customer's LLVM IR input. 2545 case Intrinsic::instrprof_increment: 2546 case Intrinsic::instrprof_increment_step: 2547 case Intrinsic::instrprof_value_profile: 2548 break; 2549 // Discard internal intrinsics. 2550 case Intrinsic::spv_value_md: 2551 break; 2552 default: { 2553 std::string DiagMsg; 2554 raw_string_ostream OS(DiagMsg); 2555 I.print(OS); 2556 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2557 report_fatal_error(DiagMsg.c_str(), false); 2558 } 2559 } 2560 return true; 2561 } 2562 2563 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 2564 const SPIRVType *ResType, 2565 MachineInstr &I) const { 2566 // there was an allocation size parameter to the allocation instruction 2567 // that is not 1 2568 MachineBasicBlock &BB = *I.getParent(); 2569 return BuildMI(BB, I, I.getDebugLoc(), 2570 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 2571 .addDef(ResVReg) 2572 .addUse(GR.getSPIRVTypeID(ResType)) 2573 .addUse(I.getOperand(2).getReg()) 2574 .constrainAllUses(TII, TRI, RBI); 2575 } 2576 2577 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 2578 const SPIRVType *ResType, 2579 MachineInstr &I) const { 2580 // Change order of instructions if needed: all OpVariable instructions in a 2581 // function must be the first instructions in the first block 2582 MachineFunction *MF = I.getParent()->getParent(); 2583 MachineBasicBlock *MBB = &MF->front(); 2584 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 2585 bool IsHeader = false; 2586 unsigned Opcode; 2587 for (; It != E && It != I; ++It) { 2588 Opcode = It->getOpcode(); 2589 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 2590 IsHeader = true; 2591 } else if (IsHeader && 2592 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 2593 ++It; 2594 break; 2595 } 2596 } 2597 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 2598 .addDef(ResVReg) 2599 .addUse(GR.getSPIRVTypeID(ResType)) 2600 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 2601 .constrainAllUses(TII, TRI, RBI); 2602 } 2603 2604 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 2605 // InstructionSelector walks backwards through the instructions. We can use 2606 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 2607 // first, so can generate an OpBranchConditional here. If there is no 2608 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 2609 const MachineInstr *PrevI = I.getPrevNode(); 2610 MachineBasicBlock &MBB = *I.getParent(); 2611 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 2612 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2613 .addUse(PrevI->getOperand(0).getReg()) 2614 .addMBB(PrevI->getOperand(1).getMBB()) 2615 .addMBB(I.getOperand(0).getMBB()) 2616 .constrainAllUses(TII, TRI, RBI); 2617 } 2618 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 2619 .addMBB(I.getOperand(0).getMBB()) 2620 .constrainAllUses(TII, TRI, RBI); 2621 } 2622 2623 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 2624 // InstructionSelector walks backwards through the instructions. For an 2625 // explicit conditional branch with no fallthrough, we use both a G_BR and a 2626 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 2627 // generate the OpBranchConditional in selectBranch above. 2628 // 2629 // If an OpBranchConditional has been generated, we simply return, as the work 2630 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 2631 // implicit fallthrough to the next basic block, so we need to create an 2632 // OpBranchConditional with an explicit "false" argument pointing to the next 2633 // basic block that LLVM would fall through to. 2634 const MachineInstr *NextI = I.getNextNode(); 2635 // Check if this has already been successfully selected. 2636 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 2637 return true; 2638 // Must be relying on implicit block fallthrough, so generate an 2639 // OpBranchConditional with the "next" basic block as the "false" target. 2640 MachineBasicBlock &MBB = *I.getParent(); 2641 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 2642 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 2643 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2644 .addUse(I.getOperand(0).getReg()) 2645 .addMBB(I.getOperand(1).getMBB()) 2646 .addMBB(NextMBB) 2647 .constrainAllUses(TII, TRI, RBI); 2648 } 2649 2650 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 2651 const SPIRVType *ResType, 2652 MachineInstr &I) const { 2653 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 2654 .addDef(ResVReg) 2655 .addUse(GR.getSPIRVTypeID(ResType)); 2656 const unsigned NumOps = I.getNumOperands(); 2657 for (unsigned i = 1; i < NumOps; i += 2) { 2658 MIB.addUse(I.getOperand(i + 0).getReg()); 2659 MIB.addMBB(I.getOperand(i + 1).getMBB()); 2660 } 2661 return MIB.constrainAllUses(TII, TRI, RBI); 2662 } 2663 2664 bool SPIRVInstructionSelector::selectGlobalValue( 2665 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 2666 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 2667 MachineIRBuilder MIRBuilder(I); 2668 const GlobalValue *GV = I.getOperand(1).getGlobal(); 2669 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 2670 SPIRVType *PointerBaseType; 2671 if (GVType->isArrayTy()) { 2672 SPIRVType *ArrayElementType = 2673 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 2674 SPIRV::AccessQualifier::ReadWrite, false); 2675 PointerBaseType = GR.getOrCreateSPIRVArrayType( 2676 ArrayElementType, GVType->getArrayNumElements(), I, TII); 2677 } else { 2678 PointerBaseType = GR.getOrCreateSPIRVType( 2679 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 2680 } 2681 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 2682 PointerBaseType, I, TII, 2683 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 2684 2685 std::string GlobalIdent; 2686 if (!GV->hasName()) { 2687 unsigned &ID = UnnamedGlobalIDs[GV]; 2688 if (ID == 0) 2689 ID = UnnamedGlobalIDs.size(); 2690 GlobalIdent = "__unnamed_" + Twine(ID).str(); 2691 } else { 2692 GlobalIdent = GV->getGlobalIdentifier(); 2693 } 2694 2695 // Behaviour of functions as operands depends on availability of the 2696 // corresponding extension (SPV_INTEL_function_pointers): 2697 // - If there is an extension to operate with functions as operands: 2698 // We create a proper constant operand and evaluate a correct type for a 2699 // function pointer. 2700 // - Without the required extension: 2701 // We have functions as operands in tests with blocks of instruction e.g. in 2702 // transcoding/global_block.ll. These operands are not used and should be 2703 // substituted by zero constants. Their type is expected to be always 2704 // OpTypePointer Function %uchar. 2705 if (isa<Function>(GV)) { 2706 const Constant *ConstVal = GV; 2707 MachineBasicBlock &BB = *I.getParent(); 2708 Register NewReg = GR.find(ConstVal, GR.CurMF); 2709 if (!NewReg.isValid()) { 2710 Register NewReg = ResVReg; 2711 GR.add(ConstVal, GR.CurMF, NewReg); 2712 const Function *GVFun = 2713 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 2714 ? dyn_cast<Function>(GV) 2715 : nullptr; 2716 if (GVFun) { 2717 // References to a function via function pointers generate virtual 2718 // registers without a definition. We will resolve it later, during 2719 // module analysis stage. 2720 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2721 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2722 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 2723 MachineInstrBuilder MB = 2724 BuildMI(BB, I, I.getDebugLoc(), 2725 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 2726 .addDef(NewReg) 2727 .addUse(GR.getSPIRVTypeID(ResType)) 2728 .addUse(FuncVReg); 2729 // mapping the function pointer to the used Function 2730 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 2731 return MB.constrainAllUses(TII, TRI, RBI); 2732 } 2733 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2734 .addDef(NewReg) 2735 .addUse(GR.getSPIRVTypeID(ResType)) 2736 .constrainAllUses(TII, TRI, RBI); 2737 } 2738 assert(NewReg != ResVReg); 2739 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2740 .addDef(ResVReg) 2741 .addUse(NewReg) 2742 .constrainAllUses(TII, TRI, RBI); 2743 } 2744 auto GlobalVar = cast<GlobalVariable>(GV); 2745 assert(GlobalVar->getName() != "llvm.global.annotations"); 2746 2747 bool HasInit = GlobalVar->hasInitializer() && 2748 !isa<UndefValue>(GlobalVar->getInitializer()); 2749 // Skip empty declaration for GVs with initilaizers till we get the decl with 2750 // passed initializer. 2751 if (HasInit && !Init) 2752 return true; 2753 2754 unsigned AddrSpace = GV->getAddressSpace(); 2755 SPIRV::StorageClass::StorageClass Storage = 2756 addressSpaceToStorageClass(AddrSpace, STI); 2757 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 2758 Storage != SPIRV::StorageClass::Function; 2759 SPIRV::LinkageType::LinkageType LnkType = 2760 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 2761 ? SPIRV::LinkageType::Import 2762 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 2763 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 2764 ? SPIRV::LinkageType::LinkOnceODR 2765 : SPIRV::LinkageType::Export); 2766 2767 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 2768 Storage, Init, GlobalVar->isConstant(), 2769 HasLnkTy, LnkType, MIRBuilder, true); 2770 return Reg.isValid(); 2771 } 2772 2773 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 2774 const SPIRVType *ResType, 2775 MachineInstr &I) const { 2776 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 2777 return selectExtInst(ResVReg, ResType, I, CL::log10); 2778 } 2779 2780 // There is no log10 instruction in the GLSL Extended Instruction set, so it 2781 // is implemented as: 2782 // log10(x) = log2(x) * (1 / log2(10)) 2783 // = log2(x) * 0.30103 2784 2785 MachineIRBuilder MIRBuilder(I); 2786 MachineBasicBlock &BB = *I.getParent(); 2787 2788 // Build log2(x). 2789 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 2790 bool Result = 2791 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2792 .addDef(VarReg) 2793 .addUse(GR.getSPIRVTypeID(ResType)) 2794 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2795 .addImm(GL::Log2) 2796 .add(I.getOperand(1)) 2797 .constrainAllUses(TII, TRI, RBI); 2798 2799 // Build 0.30103. 2800 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 2801 ResType->getOpcode() == SPIRV::OpTypeFloat); 2802 // TODO: Add matrix implementation once supported by the HLSL frontend. 2803 const SPIRVType *SpirvScalarType = 2804 ResType->getOpcode() == SPIRV::OpTypeVector 2805 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 2806 : ResType; 2807 Register ScaleReg = 2808 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 2809 2810 // Multiply log2(x) by 0.30103 to get log10(x) result. 2811 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 2812 ? SPIRV::OpVectorTimesScalar 2813 : SPIRV::OpFMulS; 2814 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2815 .addDef(ResVReg) 2816 .addUse(GR.getSPIRVTypeID(ResType)) 2817 .addUse(VarReg) 2818 .addUse(ScaleReg) 2819 .constrainAllUses(TII, TRI, RBI); 2820 2821 return Result; 2822 } 2823 2824 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 2825 const SPIRVType *ResType, 2826 MachineInstr &I) const { 2827 // DX intrinsic: @llvm.dx.thread.id(i32) 2828 // ID Name Description 2829 // 93 ThreadId reads the thread ID 2830 2831 MachineIRBuilder MIRBuilder(I); 2832 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 2833 const SPIRVType *Vec3Ty = 2834 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 2835 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 2836 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 2837 2838 // Create new register for GlobalInvocationID builtin variable. 2839 Register NewRegister = 2840 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 2841 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 2842 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 2843 2844 // Build GlobalInvocationID global variable with the necessary decorations. 2845 Register Variable = GR.buildGlobalVariable( 2846 NewRegister, PtrType, 2847 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 2848 SPIRV::StorageClass::Input, nullptr, true, true, 2849 SPIRV::LinkageType::Import, MIRBuilder, false); 2850 2851 // Create new register for loading value. 2852 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2853 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2854 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 2855 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 2856 2857 // Load v3uint value from the global variable. 2858 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2859 .addDef(LoadedRegister) 2860 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 2861 .addUse(Variable); 2862 2863 // Get Thread ID index. Expecting operand is a constant immediate value, 2864 // wrapped in a type assignment. 2865 assert(I.getOperand(2).isReg()); 2866 Register ThreadIdReg = I.getOperand(2).getReg(); 2867 SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg); 2868 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 2869 ConstTy->getOperand(1).isReg()); 2870 Register ConstReg = ConstTy->getOperand(1).getReg(); 2871 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 2872 assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); 2873 const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue(); 2874 const uint32_t ThreadId = Val.getZExtValue(); 2875 2876 // Extract the thread ID from the loaded vector value. 2877 MachineBasicBlock &BB = *I.getParent(); 2878 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2879 .addDef(ResVReg) 2880 .addUse(GR.getSPIRVTypeID(ResType)) 2881 .addUse(LoadedRegister) 2882 .addImm(ThreadId); 2883 return MIB.constrainAllUses(TII, TRI, RBI); 2884 } 2885 2886 namespace llvm { 2887 InstructionSelector * 2888 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 2889 const SPIRVSubtarget &Subtarget, 2890 const RegisterBankInfo &RBI) { 2891 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 2892 } 2893 } // namespace llvm 2894