1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/ADT/StringExtras.h" 26 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 27 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 28 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 29 #include "llvm/CodeGen/MachineInstrBuilder.h" 30 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 31 #include "llvm/CodeGen/MachineRegisterInfo.h" 32 #include "llvm/CodeGen/Register.h" 33 #include "llvm/CodeGen/TargetOpcodes.h" 34 #include "llvm/IR/IntrinsicsSPIRV.h" 35 #include "llvm/Support/Debug.h" 36 37 #define DEBUG_TYPE "spirv-isel" 38 39 using namespace llvm; 40 namespace CL = SPIRV::OpenCLExtInst; 41 namespace GL = SPIRV::GLSLExtInst; 42 43 using ExtInstList = 44 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 45 46 namespace { 47 48 #define GET_GLOBALISEL_PREDICATE_BITSET 49 #include "SPIRVGenGlobalISel.inc" 50 #undef GET_GLOBALISEL_PREDICATE_BITSET 51 52 class SPIRVInstructionSelector : public InstructionSelector { 53 const SPIRVSubtarget &STI; 54 const SPIRVInstrInfo &TII; 55 const SPIRVRegisterInfo &TRI; 56 const RegisterBankInfo &RBI; 57 SPIRVGlobalRegistry &GR; 58 MachineRegisterInfo *MRI; 59 MachineFunction *HasVRegsReset = nullptr; 60 61 /// We need to keep track of the number we give to anonymous global values to 62 /// generate the same name every time when this is needed. 63 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 64 65 public: 66 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 67 const SPIRVSubtarget &ST, 68 const RegisterBankInfo &RBI); 69 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 70 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 71 BlockFrequencyInfo *BFI) override; 72 // Common selection code. Instruction-specific selection occurs in spvSelect. 73 bool select(MachineInstr &I) override; 74 static const char *getName() { return DEBUG_TYPE; } 75 76 #define GET_GLOBALISEL_PREDICATES_DECL 77 #include "SPIRVGenGlobalISel.inc" 78 #undef GET_GLOBALISEL_PREDICATES_DECL 79 80 #define GET_GLOBALISEL_TEMPORARIES_DECL 81 #include "SPIRVGenGlobalISel.inc" 82 #undef GET_GLOBALISEL_TEMPORARIES_DECL 83 84 private: 85 void resetVRegsType(MachineFunction &MF); 86 87 // tblgen-erated 'select' implementation, used as the initial selector for 88 // the patterns that don't require complex C++. 89 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 90 91 // All instruction-specific selection that didn't happen in "select()". 92 // Is basically a large Switch/Case delegating to all other select method. 93 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 94 MachineInstr &I) const; 95 96 bool selectFirstBitHigh(Register ResVReg, const SPIRVType *ResType, 97 MachineInstr &I, bool IsSigned) const; 98 99 bool selectFirstBitHigh16(Register ResVReg, const SPIRVType *ResType, 100 MachineInstr &I, bool IsSigned) const; 101 102 bool selectFirstBitHigh32(Register ResVReg, const SPIRVType *ResType, 103 MachineInstr &I, Register SrcReg, 104 bool IsSigned) const; 105 106 bool selectFirstBitHigh64(Register ResVReg, const SPIRVType *ResType, 107 MachineInstr &I, bool IsSigned) const; 108 109 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 110 const MachineInstr *Init = nullptr) const; 111 112 bool selectNAryOpWithSrcs(Register ResVReg, const SPIRVType *ResType, 113 MachineInstr &I, std::vector<Register> SrcRegs, 114 unsigned Opcode) const; 115 116 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 117 MachineInstr &I, Register SrcReg, 118 unsigned Opcode) const; 119 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 120 unsigned Opcode) const; 121 122 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 123 MachineInstr &I) const; 124 125 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 126 MachineInstr &I) const; 127 bool selectStore(MachineInstr &I) const; 128 129 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 130 MachineInstr &I) const; 131 bool selectStackRestore(MachineInstr &I) const; 132 133 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 134 135 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I, unsigned NewOpcode, 137 unsigned NegateOpcode = 0) const; 138 139 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 140 MachineInstr &I) const; 141 142 bool selectFence(MachineInstr &I) const; 143 144 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 145 MachineInstr &I) const; 146 147 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 148 MachineInstr &I, unsigned OpType) const; 149 150 bool selectAll(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 153 bool selectAny(Register ResVReg, const SPIRVType *ResType, 154 MachineInstr &I) const; 155 156 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 157 MachineInstr &I) const; 158 159 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 162 MachineInstr &I) const; 163 164 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 165 unsigned comparisonOpcode, MachineInstr &I) const; 166 bool selectCross(Register ResVReg, const SPIRVType *ResType, 167 MachineInstr &I) const; 168 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 bool selectSign(Register ResVReg, const SPIRVType *ResType, 174 MachineInstr &I) const; 175 176 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 177 MachineInstr &I) const; 178 179 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I, unsigned Opcode) const; 181 182 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I) const; 184 185 template <bool Signed> 186 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 template <bool Signed> 189 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 190 MachineInstr &I) const; 191 192 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 193 int OpIdx) const; 194 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 195 int OpIdx) const; 196 197 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 198 MachineInstr &I) const; 199 200 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 201 bool IsSigned) const; 202 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 203 bool IsSigned, unsigned Opcode) const; 204 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 205 bool IsSigned) const; 206 207 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 208 MachineInstr &I) const; 209 210 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 211 const SPIRVType *intTy, const SPIRVType *boolTy) const; 212 213 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 214 MachineInstr &I) const; 215 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 216 MachineInstr &I) const; 217 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 218 MachineInstr &I) const; 219 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 220 MachineInstr &I) const; 221 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I) const; 223 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 224 MachineInstr &I) const; 225 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 226 MachineInstr &I) const; 227 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I) const; 229 230 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 233 MachineInstr &I) const; 234 235 bool selectBranch(MachineInstr &I) const; 236 bool selectBranchCond(MachineInstr &I) const; 237 238 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 239 MachineInstr &I) const; 240 241 bool selectExtInst(Register ResVReg, const SPIRVType *RestType, 242 MachineInstr &I, GL::GLSLExtInst GLInst) const; 243 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 244 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 245 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 246 MachineInstr &I, CL::OpenCLExtInst CLInst, 247 GL::GLSLExtInst GLInst) const; 248 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 249 MachineInstr &I, const ExtInstList &ExtInsts) const; 250 251 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 252 MachineInstr &I) const; 253 254 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 255 MachineInstr &I) const; 256 257 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 258 MachineInstr &I) const; 259 260 bool selectWaveActiveCountBits(Register ResVReg, const SPIRVType *ResType, 261 MachineInstr &I) const; 262 263 bool selectWaveReadLaneAt(Register ResVReg, const SPIRVType *ResType, 264 MachineInstr &I) const; 265 266 bool selectUnmergeValues(MachineInstr &I) const; 267 268 bool selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 269 MachineInstr &I) const; 270 271 void selectReadImageIntrinsic(Register &ResVReg, const SPIRVType *ResType, 272 MachineInstr &I) const; 273 274 // Utilities 275 std::pair<Register, bool> 276 buildI32Constant(uint32_t Val, MachineInstr &I, 277 const SPIRVType *ResType = nullptr) const; 278 279 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 280 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 281 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 282 MachineInstr &I) const; 283 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 284 285 bool wrapIntoSpecConstantOp(MachineInstr &I, 286 SmallVector<Register> &CompositeArgs) const; 287 288 Register getUcharPtrTypeReg(MachineInstr &I, 289 SPIRV::StorageClass::StorageClass SC) const; 290 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 291 Register Src, Register DestType, 292 uint32_t Opcode) const; 293 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 294 SPIRVType *SrcPtrTy) const; 295 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 296 uint32_t Binding, uint32_t ArraySize, 297 Register IndexReg, bool IsNonUniform, 298 MachineIRBuilder MIRBuilder) const; 299 SPIRVType *widenTypeToVec4(const SPIRVType *Type, MachineInstr &I) const; 300 void extractSubvector(Register &ResVReg, const SPIRVType *ResType, 301 Register &ReadReg, MachineInstr &InsertionPoint) const; 302 }; 303 304 } // end anonymous namespace 305 306 #define GET_GLOBALISEL_IMPL 307 #include "SPIRVGenGlobalISel.inc" 308 #undef GET_GLOBALISEL_IMPL 309 310 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 311 const SPIRVSubtarget &ST, 312 const RegisterBankInfo &RBI) 313 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 314 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 315 #define GET_GLOBALISEL_PREDICATES_INIT 316 #include "SPIRVGenGlobalISel.inc" 317 #undef GET_GLOBALISEL_PREDICATES_INIT 318 #define GET_GLOBALISEL_TEMPORARIES_INIT 319 #include "SPIRVGenGlobalISel.inc" 320 #undef GET_GLOBALISEL_TEMPORARIES_INIT 321 { 322 } 323 324 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 325 CodeGenCoverage *CoverageInfo, 326 ProfileSummaryInfo *PSI, 327 BlockFrequencyInfo *BFI) { 328 MRI = &MF.getRegInfo(); 329 GR.setCurrentFunc(MF); 330 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 331 } 332 333 // Ensure that register classes correspond to pattern matching rules. 334 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 335 if (HasVRegsReset == &MF) 336 return; 337 HasVRegsReset = &MF; 338 339 MachineRegisterInfo &MRI = MF.getRegInfo(); 340 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 341 Register Reg = Register::index2VirtReg(I); 342 LLT RegType = MRI.getType(Reg); 343 if (RegType.isScalar()) 344 MRI.setType(Reg, LLT::scalar(64)); 345 else if (RegType.isPointer()) 346 MRI.setType(Reg, LLT::pointer(0, 64)); 347 else if (RegType.isVector()) 348 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 349 } 350 for (const auto &MBB : MF) { 351 for (const auto &MI : MBB) { 352 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 353 continue; 354 Register DstReg = MI.getOperand(0).getReg(); 355 LLT DstType = MRI.getType(DstReg); 356 Register SrcReg = MI.getOperand(1).getReg(); 357 LLT SrcType = MRI.getType(SrcReg); 358 if (DstType != SrcType) 359 MRI.setType(DstReg, MRI.getType(SrcReg)); 360 361 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 362 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 363 if (DstRC != SrcRC && SrcRC) 364 MRI.setRegClass(DstReg, SrcRC); 365 } 366 } 367 } 368 369 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 370 371 // Defined in SPIRVLegalizerInfo.cpp. 372 extern bool isTypeFoldingSupported(unsigned Opcode); 373 374 bool SPIRVInstructionSelector::select(MachineInstr &I) { 375 resetVRegsType(*I.getParent()->getParent()); 376 377 assert(I.getParent() && "Instruction should be in a basic block!"); 378 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 379 380 Register Opcode = I.getOpcode(); 381 // If it's not a GMIR instruction, we've selected it already. 382 if (!isPreISelGenericOpcode(Opcode)) { 383 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 384 Register DstReg = I.getOperand(0).getReg(); 385 Register SrcReg = I.getOperand(1).getReg(); 386 auto *Def = MRI->getVRegDef(SrcReg); 387 if (isTypeFoldingSupported(Def->getOpcode())) { 388 bool Res = selectImpl(I, *CoverageInfo); 389 LLVM_DEBUG({ 390 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 391 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 392 I.print(dbgs()); 393 } 394 }); 395 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 396 if (Res) 397 return Res; 398 } 399 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 400 MRI->replaceRegWith(SrcReg, DstReg); 401 I.removeFromParent(); 402 return true; 403 } else if (I.getNumDefs() == 1) { 404 // Make all vregs 64 bits (for SPIR-V IDs). 405 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 406 } 407 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 408 } 409 410 if (I.getNumOperands() != I.getNumExplicitOperands()) { 411 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 412 return false; 413 } 414 415 // Common code for getting return reg+type, and removing selected instr 416 // from parent occurs here. Instr-specific selection happens in spvSelect(). 417 bool HasDefs = I.getNumDefs() > 0; 418 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 419 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 420 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 421 if (spvSelect(ResVReg, ResType, I)) { 422 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 423 for (unsigned i = 0; i < I.getNumDefs(); ++i) 424 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 425 I.removeFromParent(); 426 return true; 427 } 428 return false; 429 } 430 431 static bool mayApplyGenericSelection(unsigned Opcode) { 432 switch (Opcode) { 433 case TargetOpcode::G_CONSTANT: 434 return false; 435 case TargetOpcode::G_SADDO: 436 case TargetOpcode::G_SSUBO: 437 return true; 438 } 439 return isTypeFoldingSupported(Opcode); 440 } 441 442 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 443 const SPIRVType *ResType, 444 MachineInstr &I) const { 445 const unsigned Opcode = I.getOpcode(); 446 if (mayApplyGenericSelection(Opcode)) 447 return selectImpl(I, *CoverageInfo); 448 switch (Opcode) { 449 case TargetOpcode::G_CONSTANT: 450 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 451 I); 452 case TargetOpcode::G_GLOBAL_VALUE: 453 return selectGlobalValue(ResVReg, I); 454 case TargetOpcode::G_IMPLICIT_DEF: 455 return selectOpUndef(ResVReg, ResType, I); 456 case TargetOpcode::G_FREEZE: 457 return selectFreeze(ResVReg, ResType, I); 458 459 case TargetOpcode::G_INTRINSIC: 460 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 461 case TargetOpcode::G_INTRINSIC_CONVERGENT: 462 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 463 return selectIntrinsic(ResVReg, ResType, I); 464 case TargetOpcode::G_BITREVERSE: 465 return selectBitreverse(ResVReg, ResType, I); 466 467 case TargetOpcode::G_BUILD_VECTOR: 468 return selectBuildVector(ResVReg, ResType, I); 469 case TargetOpcode::G_SPLAT_VECTOR: 470 return selectSplatVector(ResVReg, ResType, I); 471 472 case TargetOpcode::G_SHUFFLE_VECTOR: { 473 MachineBasicBlock &BB = *I.getParent(); 474 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 475 .addDef(ResVReg) 476 .addUse(GR.getSPIRVTypeID(ResType)) 477 .addUse(I.getOperand(1).getReg()) 478 .addUse(I.getOperand(2).getReg()); 479 for (auto V : I.getOperand(3).getShuffleMask()) 480 MIB.addImm(V); 481 return MIB.constrainAllUses(TII, TRI, RBI); 482 } 483 case TargetOpcode::G_MEMMOVE: 484 case TargetOpcode::G_MEMCPY: 485 case TargetOpcode::G_MEMSET: 486 return selectMemOperation(ResVReg, I); 487 488 case TargetOpcode::G_ICMP: 489 return selectICmp(ResVReg, ResType, I); 490 case TargetOpcode::G_FCMP: 491 return selectFCmp(ResVReg, ResType, I); 492 493 case TargetOpcode::G_FRAME_INDEX: 494 return selectFrameIndex(ResVReg, ResType, I); 495 496 case TargetOpcode::G_LOAD: 497 return selectLoad(ResVReg, ResType, I); 498 case TargetOpcode::G_STORE: 499 return selectStore(I); 500 501 case TargetOpcode::G_BR: 502 return selectBranch(I); 503 case TargetOpcode::G_BRCOND: 504 return selectBranchCond(I); 505 506 case TargetOpcode::G_PHI: 507 return selectPhi(ResVReg, ResType, I); 508 509 case TargetOpcode::G_FPTOSI: 510 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 511 case TargetOpcode::G_FPTOUI: 512 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 513 514 case TargetOpcode::G_SITOFP: 515 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 516 case TargetOpcode::G_UITOFP: 517 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 518 519 case TargetOpcode::G_CTPOP: 520 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 521 case TargetOpcode::G_SMIN: 522 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 523 case TargetOpcode::G_UMIN: 524 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 525 526 case TargetOpcode::G_SMAX: 527 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 528 case TargetOpcode::G_UMAX: 529 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 530 531 case TargetOpcode::G_FMA: 532 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 533 534 case TargetOpcode::G_FPOW: 535 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 536 case TargetOpcode::G_FPOWI: 537 return selectExtInst(ResVReg, ResType, I, CL::pown); 538 539 case TargetOpcode::G_FEXP: 540 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 541 case TargetOpcode::G_FEXP2: 542 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 543 544 case TargetOpcode::G_FLOG: 545 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 546 case TargetOpcode::G_FLOG2: 547 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 548 case TargetOpcode::G_FLOG10: 549 return selectLog10(ResVReg, ResType, I); 550 551 case TargetOpcode::G_FABS: 552 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 553 case TargetOpcode::G_ABS: 554 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 555 556 case TargetOpcode::G_FMINNUM: 557 case TargetOpcode::G_FMINIMUM: 558 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 559 case TargetOpcode::G_FMAXNUM: 560 case TargetOpcode::G_FMAXIMUM: 561 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 562 563 case TargetOpcode::G_FCOPYSIGN: 564 return selectExtInst(ResVReg, ResType, I, CL::copysign); 565 566 case TargetOpcode::G_FCEIL: 567 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 568 case TargetOpcode::G_FFLOOR: 569 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 570 571 case TargetOpcode::G_FCOS: 572 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 573 case TargetOpcode::G_FSIN: 574 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 575 case TargetOpcode::G_FTAN: 576 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 577 case TargetOpcode::G_FACOS: 578 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 579 case TargetOpcode::G_FASIN: 580 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 581 case TargetOpcode::G_FATAN: 582 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 583 case TargetOpcode::G_FATAN2: 584 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 585 case TargetOpcode::G_FCOSH: 586 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 587 case TargetOpcode::G_FSINH: 588 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 589 case TargetOpcode::G_FTANH: 590 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 591 592 case TargetOpcode::G_FSQRT: 593 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 594 595 case TargetOpcode::G_CTTZ: 596 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 597 return selectExtInst(ResVReg, ResType, I, CL::ctz); 598 case TargetOpcode::G_CTLZ: 599 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 600 return selectExtInst(ResVReg, ResType, I, CL::clz); 601 602 case TargetOpcode::G_INTRINSIC_ROUND: 603 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 604 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 605 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 606 case TargetOpcode::G_INTRINSIC_TRUNC: 607 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 608 case TargetOpcode::G_FRINT: 609 case TargetOpcode::G_FNEARBYINT: 610 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 611 612 case TargetOpcode::G_SMULH: 613 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 614 case TargetOpcode::G_UMULH: 615 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 616 617 case TargetOpcode::G_SADDSAT: 618 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 619 case TargetOpcode::G_UADDSAT: 620 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 621 case TargetOpcode::G_SSUBSAT: 622 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 623 case TargetOpcode::G_USUBSAT: 624 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 625 626 case TargetOpcode::G_UADDO: 627 return selectOverflowArith(ResVReg, ResType, I, 628 ResType->getOpcode() == SPIRV::OpTypeVector 629 ? SPIRV::OpIAddCarryV 630 : SPIRV::OpIAddCarryS); 631 case TargetOpcode::G_USUBO: 632 return selectOverflowArith(ResVReg, ResType, I, 633 ResType->getOpcode() == SPIRV::OpTypeVector 634 ? SPIRV::OpISubBorrowV 635 : SPIRV::OpISubBorrowS); 636 case TargetOpcode::G_UMULO: 637 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 638 case TargetOpcode::G_SMULO: 639 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 640 641 case TargetOpcode::G_SEXT: 642 return selectExt(ResVReg, ResType, I, true); 643 case TargetOpcode::G_ANYEXT: 644 case TargetOpcode::G_ZEXT: 645 return selectExt(ResVReg, ResType, I, false); 646 case TargetOpcode::G_TRUNC: 647 return selectTrunc(ResVReg, ResType, I); 648 case TargetOpcode::G_FPTRUNC: 649 case TargetOpcode::G_FPEXT: 650 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 651 652 case TargetOpcode::G_PTRTOINT: 653 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 654 case TargetOpcode::G_INTTOPTR: 655 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 656 case TargetOpcode::G_BITCAST: 657 return selectBitcast(ResVReg, ResType, I); 658 case TargetOpcode::G_ADDRSPACE_CAST: 659 return selectAddrSpaceCast(ResVReg, ResType, I); 660 case TargetOpcode::G_PTR_ADD: { 661 // Currently, we get G_PTR_ADD only applied to global variables. 662 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 663 Register GV = I.getOperand(1).getReg(); 664 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 665 (void)II; 666 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 667 (*II).getOpcode() == TargetOpcode::COPY || 668 (*II).getOpcode() == SPIRV::OpVariable) && 669 isImm(I.getOperand(2), MRI)); 670 // It may be the initialization of a global variable. 671 bool IsGVInit = false; 672 for (MachineRegisterInfo::use_instr_iterator 673 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 674 UseEnd = MRI->use_instr_end(); 675 UseIt != UseEnd; UseIt = std::next(UseIt)) { 676 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 677 (*UseIt).getOpcode() == SPIRV::OpVariable) { 678 IsGVInit = true; 679 break; 680 } 681 } 682 MachineBasicBlock &BB = *I.getParent(); 683 if (!IsGVInit) { 684 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 685 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 686 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 687 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 688 // Build a new virtual register that is associated with the required 689 // data type. 690 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 691 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 692 // Having a correctly typed base we are ready to build the actually 693 // required GEP. It may not be a constant though, because all Operands 694 // of OpSpecConstantOp is to originate from other const instructions, 695 // and only the AccessChain named opcodes accept a global OpVariable 696 // instruction. We can't use an AccessChain opcode because of the type 697 // mismatch between result and base types. 698 if (!GR.isBitcastCompatible(ResType, GVType)) 699 report_fatal_error( 700 "incompatible result and operand types in a bitcast"); 701 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 702 MachineInstrBuilder MIB = 703 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 704 .addDef(NewVReg) 705 .addUse(ResTypeReg) 706 .addUse(GV); 707 return MIB.constrainAllUses(TII, TRI, RBI) && 708 BuildMI(BB, I, I.getDebugLoc(), 709 TII.get(STI.isVulkanEnv() 710 ? SPIRV::OpInBoundsAccessChain 711 : SPIRV::OpInBoundsPtrAccessChain)) 712 .addDef(ResVReg) 713 .addUse(ResTypeReg) 714 .addUse(NewVReg) 715 .addUse(I.getOperand(2).getReg()) 716 .constrainAllUses(TII, TRI, RBI); 717 } else { 718 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 719 .addDef(ResVReg) 720 .addUse(GR.getSPIRVTypeID(ResType)) 721 .addImm( 722 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 723 .addUse(GV) 724 .addUse(I.getOperand(2).getReg()) 725 .constrainAllUses(TII, TRI, RBI); 726 } 727 } 728 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 729 // initialize a global variable with a constant expression (e.g., the test 730 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 731 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 732 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 733 .addDef(ResVReg) 734 .addUse(GR.getSPIRVTypeID(ResType)) 735 .addImm(static_cast<uint32_t>( 736 SPIRV::Opcode::InBoundsPtrAccessChain)) 737 .addUse(GV) 738 .addUse(Idx) 739 .addUse(I.getOperand(2).getReg()); 740 return MIB.constrainAllUses(TII, TRI, RBI); 741 } 742 743 case TargetOpcode::G_ATOMICRMW_OR: 744 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 745 case TargetOpcode::G_ATOMICRMW_ADD: 746 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 747 case TargetOpcode::G_ATOMICRMW_AND: 748 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 749 case TargetOpcode::G_ATOMICRMW_MAX: 750 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 751 case TargetOpcode::G_ATOMICRMW_MIN: 752 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 753 case TargetOpcode::G_ATOMICRMW_SUB: 754 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 755 case TargetOpcode::G_ATOMICRMW_XOR: 756 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 757 case TargetOpcode::G_ATOMICRMW_UMAX: 758 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 759 case TargetOpcode::G_ATOMICRMW_UMIN: 760 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 761 case TargetOpcode::G_ATOMICRMW_XCHG: 762 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 763 case TargetOpcode::G_ATOMIC_CMPXCHG: 764 return selectAtomicCmpXchg(ResVReg, ResType, I); 765 766 case TargetOpcode::G_ATOMICRMW_FADD: 767 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 768 case TargetOpcode::G_ATOMICRMW_FSUB: 769 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 770 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 771 SPIRV::OpFNegate); 772 case TargetOpcode::G_ATOMICRMW_FMIN: 773 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 774 case TargetOpcode::G_ATOMICRMW_FMAX: 775 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 776 777 case TargetOpcode::G_FENCE: 778 return selectFence(I); 779 780 case TargetOpcode::G_STACKSAVE: 781 return selectStackSave(ResVReg, ResType, I); 782 case TargetOpcode::G_STACKRESTORE: 783 return selectStackRestore(I); 784 785 case TargetOpcode::G_UNMERGE_VALUES: 786 return selectUnmergeValues(I); 787 788 // Discard gen opcodes for intrinsics which we do not expect to actually 789 // represent code after lowering or intrinsics which are not implemented but 790 // should not crash when found in a customer's LLVM IR input. 791 case TargetOpcode::G_TRAP: 792 case TargetOpcode::G_DEBUGTRAP: 793 case TargetOpcode::G_UBSANTRAP: 794 case TargetOpcode::DBG_LABEL: 795 return true; 796 797 default: 798 return false; 799 } 800 } 801 802 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 803 const SPIRVType *ResType, 804 MachineInstr &I, 805 GL::GLSLExtInst GLInst) const { 806 return selectExtInst(ResVReg, ResType, I, 807 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}); 808 } 809 810 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 811 const SPIRVType *ResType, 812 MachineInstr &I, 813 CL::OpenCLExtInst CLInst) const { 814 return selectExtInst(ResVReg, ResType, I, 815 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 816 } 817 818 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 819 const SPIRVType *ResType, 820 MachineInstr &I, 821 CL::OpenCLExtInst CLInst, 822 GL::GLSLExtInst GLInst) const { 823 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 824 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 825 return selectExtInst(ResVReg, ResType, I, ExtInsts); 826 } 827 828 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 829 const SPIRVType *ResType, 830 MachineInstr &I, 831 const ExtInstList &Insts) const { 832 833 for (const auto &Ex : Insts) { 834 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 835 uint32_t Opcode = Ex.second; 836 if (STI.canUseExtInstSet(Set)) { 837 MachineBasicBlock &BB = *I.getParent(); 838 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 839 .addDef(ResVReg) 840 .addUse(GR.getSPIRVTypeID(ResType)) 841 .addImm(static_cast<uint32_t>(Set)) 842 .addImm(Opcode); 843 const unsigned NumOps = I.getNumOperands(); 844 unsigned Index = 1; 845 if (Index < NumOps && 846 I.getOperand(Index).getType() == 847 MachineOperand::MachineOperandType::MO_IntrinsicID) 848 Index = 2; 849 for (; Index < NumOps; ++Index) 850 MIB.add(I.getOperand(Index)); 851 return MIB.constrainAllUses(TII, TRI, RBI); 852 } 853 } 854 return false; 855 } 856 857 bool SPIRVInstructionSelector::selectNAryOpWithSrcs(Register ResVReg, 858 const SPIRVType *ResType, 859 MachineInstr &I, 860 std::vector<Register> Srcs, 861 unsigned Opcode) const { 862 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 863 .addDef(ResVReg) 864 .addUse(GR.getSPIRVTypeID(ResType)); 865 for (Register SReg : Srcs) { 866 MIB.addUse(SReg); 867 } 868 return MIB.constrainAllUses(TII, TRI, RBI); 869 } 870 871 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 872 const SPIRVType *ResType, 873 MachineInstr &I, 874 Register SrcReg, 875 unsigned Opcode) const { 876 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 877 .addDef(ResVReg) 878 .addUse(GR.getSPIRVTypeID(ResType)) 879 .addUse(SrcReg) 880 .constrainAllUses(TII, TRI, RBI); 881 } 882 883 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 884 const SPIRVType *ResType, 885 MachineInstr &I, 886 unsigned Opcode) const { 887 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 888 Register SrcReg = I.getOperand(1).getReg(); 889 bool IsGV = false; 890 for (MachineRegisterInfo::def_instr_iterator DefIt = 891 MRI->def_instr_begin(SrcReg); 892 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 893 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 894 IsGV = true; 895 break; 896 } 897 } 898 if (IsGV) { 899 uint32_t SpecOpcode = 0; 900 switch (Opcode) { 901 case SPIRV::OpConvertPtrToU: 902 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 903 break; 904 case SPIRV::OpConvertUToPtr: 905 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 906 break; 907 } 908 if (SpecOpcode) 909 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 910 TII.get(SPIRV::OpSpecConstantOp)) 911 .addDef(ResVReg) 912 .addUse(GR.getSPIRVTypeID(ResType)) 913 .addImm(SpecOpcode) 914 .addUse(SrcReg) 915 .constrainAllUses(TII, TRI, RBI); 916 } 917 } 918 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 919 Opcode); 920 } 921 922 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 923 const SPIRVType *ResType, 924 MachineInstr &I) const { 925 Register OpReg = I.getOperand(1).getReg(); 926 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 927 if (!GR.isBitcastCompatible(ResType, OpType)) 928 report_fatal_error("incompatible result and operand types in a bitcast"); 929 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 930 } 931 932 static void addMemoryOperands(MachineMemOperand *MemOp, 933 MachineInstrBuilder &MIB) { 934 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 935 if (MemOp->isVolatile()) 936 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 937 if (MemOp->isNonTemporal()) 938 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 939 if (MemOp->getAlign().value()) 940 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 941 942 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 943 MIB.addImm(SpvMemOp); 944 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 945 MIB.addImm(MemOp->getAlign().value()); 946 } 947 } 948 949 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 950 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 951 if (Flags & MachineMemOperand::Flags::MOVolatile) 952 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 953 if (Flags & MachineMemOperand::Flags::MONonTemporal) 954 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 955 956 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 957 MIB.addImm(SpvMemOp); 958 } 959 960 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 961 const SPIRVType *ResType, 962 MachineInstr &I) const { 963 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 964 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 965 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 966 .addDef(ResVReg) 967 .addUse(GR.getSPIRVTypeID(ResType)) 968 .addUse(Ptr); 969 if (!I.getNumMemOperands()) { 970 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 971 I.getOpcode() == 972 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 973 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 974 } else { 975 addMemoryOperands(*I.memoperands_begin(), MIB); 976 } 977 return MIB.constrainAllUses(TII, TRI, RBI); 978 } 979 980 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 981 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 982 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 983 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 984 MachineBasicBlock &BB = *I.getParent(); 985 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 986 .addUse(Ptr) 987 .addUse(StoreVal); 988 if (!I.getNumMemOperands()) { 989 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 990 I.getOpcode() == 991 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 992 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 993 } else { 994 addMemoryOperands(*I.memoperands_begin(), MIB); 995 } 996 return MIB.constrainAllUses(TII, TRI, RBI); 997 } 998 999 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 1000 const SPIRVType *ResType, 1001 MachineInstr &I) const { 1002 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1003 report_fatal_error( 1004 "llvm.stacksave intrinsic: this instruction requires the following " 1005 "SPIR-V extension: SPV_INTEL_variable_length_array", 1006 false); 1007 MachineBasicBlock &BB = *I.getParent(); 1008 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 1009 .addDef(ResVReg) 1010 .addUse(GR.getSPIRVTypeID(ResType)) 1011 .constrainAllUses(TII, TRI, RBI); 1012 } 1013 1014 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 1015 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 1016 report_fatal_error( 1017 "llvm.stackrestore intrinsic: this instruction requires the following " 1018 "SPIR-V extension: SPV_INTEL_variable_length_array", 1019 false); 1020 if (!I.getOperand(0).isReg()) 1021 return false; 1022 MachineBasicBlock &BB = *I.getParent(); 1023 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 1024 .addUse(I.getOperand(0).getReg()) 1025 .constrainAllUses(TII, TRI, RBI); 1026 } 1027 1028 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 1029 MachineInstr &I) const { 1030 MachineBasicBlock &BB = *I.getParent(); 1031 Register SrcReg = I.getOperand(1).getReg(); 1032 bool Result = true; 1033 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 1034 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 1035 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 1036 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 1037 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1038 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 1039 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 1040 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 1041 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 1042 // TODO: check if we have such GV, add init, use buildGlobalVariable. 1043 Function &CurFunction = GR.CurMF->getFunction(); 1044 Type *LLVMArrTy = 1045 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 1046 // Module takes ownership of the global var. 1047 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 1048 true, GlobalValue::InternalLinkage, 1049 Constant::getNullValue(LLVMArrTy)); 1050 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1051 GR.add(GV, GR.CurMF, VarReg); 1052 1053 Result &= 1054 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1055 .addDef(VarReg) 1056 .addUse(GR.getSPIRVTypeID(VarTy)) 1057 .addImm(SPIRV::StorageClass::UniformConstant) 1058 .addUse(Const) 1059 .constrainAllUses(TII, TRI, RBI); 1060 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1061 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1062 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1063 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1064 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 1065 } 1066 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1067 .addUse(I.getOperand(0).getReg()) 1068 .addUse(SrcReg) 1069 .addUse(I.getOperand(2).getReg()); 1070 if (I.getNumMemOperands()) 1071 addMemoryOperands(*I.memoperands_begin(), MIB); 1072 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1073 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1074 Result &= 1075 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1076 .addUse(MIB->getOperand(0).getReg()) 1077 .constrainAllUses(TII, TRI, RBI); 1078 return Result; 1079 } 1080 1081 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1082 const SPIRVType *ResType, 1083 MachineInstr &I, 1084 unsigned NewOpcode, 1085 unsigned NegateOpcode) const { 1086 bool Result = true; 1087 assert(I.hasOneMemOperand()); 1088 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1089 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1090 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1091 auto ScopeConstant = buildI32Constant(Scope, I); 1092 Register ScopeReg = ScopeConstant.first; 1093 Result &= ScopeConstant.second; 1094 1095 Register Ptr = I.getOperand(1).getReg(); 1096 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1097 // auto ScSem = 1098 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1099 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1100 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1101 auto MemSemConstant = buildI32Constant(MemSem /*| ScSem*/, I); 1102 Register MemSemReg = MemSemConstant.first; 1103 Result &= MemSemConstant.second; 1104 1105 Register ValueReg = I.getOperand(2).getReg(); 1106 if (NegateOpcode != 0) { 1107 // Translation with negative value operand is requested 1108 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1109 Result &= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1110 ValueReg = TmpReg; 1111 } 1112 1113 return Result && 1114 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1115 .addDef(ResVReg) 1116 .addUse(GR.getSPIRVTypeID(ResType)) 1117 .addUse(Ptr) 1118 .addUse(ScopeReg) 1119 .addUse(MemSemReg) 1120 .addUse(ValueReg) 1121 .constrainAllUses(TII, TRI, RBI); 1122 } 1123 1124 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1125 unsigned ArgI = I.getNumOperands() - 1; 1126 Register SrcReg = 1127 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1128 SPIRVType *DefType = 1129 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1130 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1131 report_fatal_error( 1132 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1133 1134 SPIRVType *ScalarType = 1135 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1136 MachineBasicBlock &BB = *I.getParent(); 1137 bool Res = false; 1138 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1139 Register ResVReg = I.getOperand(i).getReg(); 1140 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1141 if (!ResType) { 1142 // There was no "assign type" actions, let's fix this now 1143 ResType = ScalarType; 1144 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1145 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1146 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1147 } 1148 auto MIB = 1149 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1150 .addDef(ResVReg) 1151 .addUse(GR.getSPIRVTypeID(ResType)) 1152 .addUse(SrcReg) 1153 .addImm(static_cast<int64_t>(i)); 1154 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1155 } 1156 return Res; 1157 } 1158 1159 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1160 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1161 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1162 auto MemSemConstant = buildI32Constant(MemSem, I); 1163 Register MemSemReg = MemSemConstant.first; 1164 bool Result = MemSemConstant.second; 1165 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1166 uint32_t Scope = static_cast<uint32_t>( 1167 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1168 auto ScopeConstant = buildI32Constant(Scope, I); 1169 Register ScopeReg = ScopeConstant.first; 1170 Result &= ScopeConstant.second; 1171 MachineBasicBlock &BB = *I.getParent(); 1172 return Result && 1173 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1174 .addUse(ScopeReg) 1175 .addUse(MemSemReg) 1176 .constrainAllUses(TII, TRI, RBI); 1177 } 1178 1179 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1180 const SPIRVType *ResType, 1181 MachineInstr &I, 1182 unsigned Opcode) const { 1183 Type *ResTy = nullptr; 1184 StringRef ResName; 1185 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1186 report_fatal_error( 1187 "Not enough info to select the arithmetic with overflow instruction"); 1188 if (!ResTy || !ResTy->isStructTy()) 1189 report_fatal_error("Expect struct type result for the arithmetic " 1190 "with overflow instruction"); 1191 // "Result Type must be from OpTypeStruct. The struct must have two members, 1192 // and the two members must be the same type." 1193 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1194 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1195 // Build SPIR-V types and constant(s) if needed. 1196 MachineIRBuilder MIRBuilder(I); 1197 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1198 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1199 assert(I.getNumDefs() > 1 && "Not enought operands"); 1200 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1201 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1202 if (N > 1) 1203 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1204 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1205 Register ZeroReg = buildZerosVal(ResType, I); 1206 // A new virtual register to store the result struct. 1207 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1208 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1209 // Build the result name if needed. 1210 if (ResName.size() > 0) 1211 buildOpName(StructVReg, ResName, MIRBuilder); 1212 // Build the arithmetic with overflow instruction. 1213 MachineBasicBlock &BB = *I.getParent(); 1214 auto MIB = 1215 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1216 .addDef(StructVReg) 1217 .addUse(GR.getSPIRVTypeID(StructType)); 1218 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1219 MIB.addUse(I.getOperand(i).getReg()); 1220 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1221 // Build instructions to extract fields of the instruction's result. 1222 // A new virtual register to store the higher part of the result struct. 1223 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1224 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1225 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1226 auto MIB = 1227 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1228 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1229 .addUse(GR.getSPIRVTypeID(ResType)) 1230 .addUse(StructVReg) 1231 .addImm(i); 1232 Result &= MIB.constrainAllUses(TII, TRI, RBI); 1233 } 1234 // Build boolean value from the higher part. 1235 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1236 .addDef(I.getOperand(1).getReg()) 1237 .addUse(BoolTypeReg) 1238 .addUse(HigherVReg) 1239 .addUse(ZeroReg) 1240 .constrainAllUses(TII, TRI, RBI); 1241 } 1242 1243 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1244 const SPIRVType *ResType, 1245 MachineInstr &I) const { 1246 bool Result = true; 1247 Register ScopeReg; 1248 Register MemSemEqReg; 1249 Register MemSemNeqReg; 1250 Register Ptr = I.getOperand(2).getReg(); 1251 if (!isa<GIntrinsic>(I)) { 1252 assert(I.hasOneMemOperand()); 1253 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1254 unsigned Scope = static_cast<uint32_t>(getMemScope( 1255 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1256 auto ScopeConstant = buildI32Constant(Scope, I); 1257 ScopeReg = ScopeConstant.first; 1258 Result &= ScopeConstant.second; 1259 1260 unsigned ScSem = static_cast<uint32_t>( 1261 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1262 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1263 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1264 auto MemSemEqConstant = buildI32Constant(MemSemEq, I); 1265 MemSemEqReg = MemSemEqConstant.first; 1266 Result &= MemSemEqConstant.second; 1267 AtomicOrdering FO = MemOp->getFailureOrdering(); 1268 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1269 if (MemSemEq == MemSemNeq) 1270 MemSemNeqReg = MemSemEqReg; 1271 else { 1272 auto MemSemNeqConstant = buildI32Constant(MemSemEq, I); 1273 MemSemNeqReg = MemSemNeqConstant.first; 1274 Result &= MemSemNeqConstant.second; 1275 } 1276 } else { 1277 ScopeReg = I.getOperand(5).getReg(); 1278 MemSemEqReg = I.getOperand(6).getReg(); 1279 MemSemNeqReg = I.getOperand(7).getReg(); 1280 } 1281 1282 Register Cmp = I.getOperand(3).getReg(); 1283 Register Val = I.getOperand(4).getReg(); 1284 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1285 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1286 const DebugLoc &DL = I.getDebugLoc(); 1287 Result &= 1288 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1289 .addDef(ACmpRes) 1290 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1291 .addUse(Ptr) 1292 .addUse(ScopeReg) 1293 .addUse(MemSemEqReg) 1294 .addUse(MemSemNeqReg) 1295 .addUse(Val) 1296 .addUse(Cmp) 1297 .constrainAllUses(TII, TRI, RBI); 1298 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1299 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1300 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1301 .addDef(CmpSuccReg) 1302 .addUse(GR.getSPIRVTypeID(BoolTy)) 1303 .addUse(ACmpRes) 1304 .addUse(Cmp) 1305 .constrainAllUses(TII, TRI, RBI); 1306 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1307 Result &= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1308 .addDef(TmpReg) 1309 .addUse(GR.getSPIRVTypeID(ResType)) 1310 .addUse(ACmpRes) 1311 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1312 .addImm(0) 1313 .constrainAllUses(TII, TRI, RBI); 1314 return Result && 1315 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1316 .addDef(ResVReg) 1317 .addUse(GR.getSPIRVTypeID(ResType)) 1318 .addUse(CmpSuccReg) 1319 .addUse(TmpReg) 1320 .addImm(1) 1321 .constrainAllUses(TII, TRI, RBI); 1322 } 1323 1324 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1325 switch (SC) { 1326 case SPIRV::StorageClass::Workgroup: 1327 case SPIRV::StorageClass::CrossWorkgroup: 1328 case SPIRV::StorageClass::Function: 1329 return true; 1330 default: 1331 return false; 1332 } 1333 } 1334 1335 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1336 switch (SC) { 1337 case SPIRV::StorageClass::DeviceOnlyINTEL: 1338 case SPIRV::StorageClass::HostOnlyINTEL: 1339 return true; 1340 default: 1341 return false; 1342 } 1343 } 1344 1345 // Returns true ResVReg is referred only from global vars and OpName's. 1346 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1347 bool IsGRef = false; 1348 bool IsAllowedRefs = 1349 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1350 [&IsGRef](auto const &It) { 1351 unsigned Opcode = It.getOpcode(); 1352 if (Opcode == SPIRV::OpConstantComposite || 1353 Opcode == SPIRV::OpVariable || 1354 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1355 return IsGRef = true; 1356 return Opcode == SPIRV::OpName; 1357 }); 1358 return IsAllowedRefs && IsGRef; 1359 } 1360 1361 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1362 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1363 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1364 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1365 } 1366 1367 MachineInstrBuilder 1368 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1369 Register Src, Register DestType, 1370 uint32_t Opcode) const { 1371 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1372 TII.get(SPIRV::OpSpecConstantOp)) 1373 .addDef(Dest) 1374 .addUse(DestType) 1375 .addImm(Opcode) 1376 .addUse(Src); 1377 } 1378 1379 MachineInstrBuilder 1380 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1381 SPIRVType *SrcPtrTy) const { 1382 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1383 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1384 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1385 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1386 SPIRV::StorageClass::Generic), 1387 GR.getPointerSize())); 1388 MachineFunction *MF = I.getParent()->getParent(); 1389 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1390 MachineInstrBuilder MIB = buildSpecConstantOp( 1391 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1392 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1393 GR.add(MIB.getInstr(), MF, Tmp); 1394 return MIB; 1395 } 1396 1397 // In SPIR-V address space casting can only happen to and from the Generic 1398 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1399 // pointers to and from Generic pointers. As such, we can convert e.g. from 1400 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1401 // other combinations can only be done by a bitcast, and are probably not safe. 1402 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1403 const SPIRVType *ResType, 1404 MachineInstr &I) const { 1405 MachineBasicBlock &BB = *I.getParent(); 1406 const DebugLoc &DL = I.getDebugLoc(); 1407 1408 Register SrcPtr = I.getOperand(1).getReg(); 1409 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1410 1411 // don't generate a cast for a null that may be represented by OpTypeInt 1412 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1413 ResType->getOpcode() != SPIRV::OpTypePointer) 1414 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1415 .addDef(ResVReg) 1416 .addUse(SrcPtr) 1417 .constrainAllUses(TII, TRI, RBI); 1418 1419 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1420 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1421 1422 if (isASCastInGVar(MRI, ResVReg)) { 1423 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1424 // are expressed by OpSpecConstantOp with an Opcode. 1425 // TODO: maybe insert a check whether the Kernel capability was declared and 1426 // so PtrCastToGeneric/GenericCastToPtr are available. 1427 unsigned SpecOpcode = 1428 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1429 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1430 : (SrcSC == SPIRV::StorageClass::Generic && 1431 isGenericCastablePtr(DstSC) 1432 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1433 : 0); 1434 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1435 // correct value of ResType and use general i8* instead. Maybe this should 1436 // be addressed in the emit-intrinsic step to infer a correct 1437 // OpConstantComposite type. 1438 if (SpecOpcode) { 1439 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1440 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1441 .constrainAllUses(TII, TRI, RBI); 1442 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1443 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1444 return MIB.constrainAllUses(TII, TRI, RBI) && 1445 buildSpecConstantOp( 1446 I, ResVReg, MIB->getOperand(0).getReg(), 1447 getUcharPtrTypeReg(I, DstSC), 1448 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1449 .constrainAllUses(TII, TRI, RBI); 1450 } 1451 } 1452 1453 // don't generate a cast between identical storage classes 1454 if (SrcSC == DstSC) 1455 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1456 .addDef(ResVReg) 1457 .addUse(SrcPtr) 1458 .constrainAllUses(TII, TRI, RBI); 1459 1460 // Casting from an eligible pointer to Generic. 1461 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1462 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1463 // Casting from Generic to an eligible pointer. 1464 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1465 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1466 // Casting between 2 eligible pointers using Generic as an intermediary. 1467 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1468 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1469 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1470 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1471 bool Result = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1472 .addDef(Tmp) 1473 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1474 .addUse(SrcPtr) 1475 .constrainAllUses(TII, TRI, RBI); 1476 return Result && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1477 .addDef(ResVReg) 1478 .addUse(GR.getSPIRVTypeID(ResType)) 1479 .addUse(Tmp) 1480 .constrainAllUses(TII, TRI, RBI); 1481 } 1482 1483 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1484 // be applied 1485 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1486 return selectUnOp(ResVReg, ResType, I, 1487 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1488 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1489 return selectUnOp(ResVReg, ResType, I, 1490 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1491 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1492 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1493 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1494 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1495 1496 // Bitcast for pointers requires that the address spaces must match 1497 return false; 1498 } 1499 1500 static unsigned getFCmpOpcode(unsigned PredNum) { 1501 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1502 switch (Pred) { 1503 case CmpInst::FCMP_OEQ: 1504 return SPIRV::OpFOrdEqual; 1505 case CmpInst::FCMP_OGE: 1506 return SPIRV::OpFOrdGreaterThanEqual; 1507 case CmpInst::FCMP_OGT: 1508 return SPIRV::OpFOrdGreaterThan; 1509 case CmpInst::FCMP_OLE: 1510 return SPIRV::OpFOrdLessThanEqual; 1511 case CmpInst::FCMP_OLT: 1512 return SPIRV::OpFOrdLessThan; 1513 case CmpInst::FCMP_ONE: 1514 return SPIRV::OpFOrdNotEqual; 1515 case CmpInst::FCMP_ORD: 1516 return SPIRV::OpOrdered; 1517 case CmpInst::FCMP_UEQ: 1518 return SPIRV::OpFUnordEqual; 1519 case CmpInst::FCMP_UGE: 1520 return SPIRV::OpFUnordGreaterThanEqual; 1521 case CmpInst::FCMP_UGT: 1522 return SPIRV::OpFUnordGreaterThan; 1523 case CmpInst::FCMP_ULE: 1524 return SPIRV::OpFUnordLessThanEqual; 1525 case CmpInst::FCMP_ULT: 1526 return SPIRV::OpFUnordLessThan; 1527 case CmpInst::FCMP_UNE: 1528 return SPIRV::OpFUnordNotEqual; 1529 case CmpInst::FCMP_UNO: 1530 return SPIRV::OpUnordered; 1531 default: 1532 llvm_unreachable("Unknown predicate type for FCmp"); 1533 } 1534 } 1535 1536 static unsigned getICmpOpcode(unsigned PredNum) { 1537 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1538 switch (Pred) { 1539 case CmpInst::ICMP_EQ: 1540 return SPIRV::OpIEqual; 1541 case CmpInst::ICMP_NE: 1542 return SPIRV::OpINotEqual; 1543 case CmpInst::ICMP_SGE: 1544 return SPIRV::OpSGreaterThanEqual; 1545 case CmpInst::ICMP_SGT: 1546 return SPIRV::OpSGreaterThan; 1547 case CmpInst::ICMP_SLE: 1548 return SPIRV::OpSLessThanEqual; 1549 case CmpInst::ICMP_SLT: 1550 return SPIRV::OpSLessThan; 1551 case CmpInst::ICMP_UGE: 1552 return SPIRV::OpUGreaterThanEqual; 1553 case CmpInst::ICMP_UGT: 1554 return SPIRV::OpUGreaterThan; 1555 case CmpInst::ICMP_ULE: 1556 return SPIRV::OpULessThanEqual; 1557 case CmpInst::ICMP_ULT: 1558 return SPIRV::OpULessThan; 1559 default: 1560 llvm_unreachable("Unknown predicate type for ICmp"); 1561 } 1562 } 1563 1564 static unsigned getPtrCmpOpcode(unsigned Pred) { 1565 switch (static_cast<CmpInst::Predicate>(Pred)) { 1566 case CmpInst::ICMP_EQ: 1567 return SPIRV::OpPtrEqual; 1568 case CmpInst::ICMP_NE: 1569 return SPIRV::OpPtrNotEqual; 1570 default: 1571 llvm_unreachable("Unknown predicate type for pointer comparison"); 1572 } 1573 } 1574 1575 // Return the logical operation, or abort if none exists. 1576 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1577 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1578 switch (Pred) { 1579 case CmpInst::ICMP_EQ: 1580 return SPIRV::OpLogicalEqual; 1581 case CmpInst::ICMP_NE: 1582 return SPIRV::OpLogicalNotEqual; 1583 default: 1584 llvm_unreachable("Unknown predicate type for Bool comparison"); 1585 } 1586 } 1587 1588 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1589 if (!LLVMFloatTy) 1590 return APFloat::getZero(APFloat::IEEEsingle()); 1591 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1592 case Type::HalfTyID: 1593 return APFloat::getZero(APFloat::IEEEhalf()); 1594 default: 1595 case Type::FloatTyID: 1596 return APFloat::getZero(APFloat::IEEEsingle()); 1597 case Type::DoubleTyID: 1598 return APFloat::getZero(APFloat::IEEEdouble()); 1599 } 1600 } 1601 1602 static APFloat getOneFP(const Type *LLVMFloatTy) { 1603 if (!LLVMFloatTy) 1604 return APFloat::getOne(APFloat::IEEEsingle()); 1605 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1606 case Type::HalfTyID: 1607 return APFloat::getOne(APFloat::IEEEhalf()); 1608 default: 1609 case Type::FloatTyID: 1610 return APFloat::getOne(APFloat::IEEEsingle()); 1611 case Type::DoubleTyID: 1612 return APFloat::getOne(APFloat::IEEEdouble()); 1613 } 1614 } 1615 1616 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1617 const SPIRVType *ResType, 1618 MachineInstr &I, 1619 unsigned OpAnyOrAll) const { 1620 assert(I.getNumOperands() == 3); 1621 assert(I.getOperand(2).isReg()); 1622 MachineBasicBlock &BB = *I.getParent(); 1623 Register InputRegister = I.getOperand(2).getReg(); 1624 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1625 1626 if (!InputType) 1627 report_fatal_error("Input Type could not be determined."); 1628 1629 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1630 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1631 if (IsBoolTy && !IsVectorTy) { 1632 assert(ResVReg == I.getOperand(0).getReg()); 1633 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1634 TII.get(TargetOpcode::COPY)) 1635 .addDef(ResVReg) 1636 .addUse(InputRegister) 1637 .constrainAllUses(TII, TRI, RBI); 1638 } 1639 1640 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1641 unsigned SpirvNotEqualId = 1642 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1643 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1644 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1645 Register NotEqualReg = ResVReg; 1646 1647 if (IsVectorTy) { 1648 NotEqualReg = IsBoolTy ? InputRegister 1649 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1650 const unsigned NumElts = InputType->getOperand(2).getImm(); 1651 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1652 } 1653 1654 bool Result = true; 1655 if (!IsBoolTy) { 1656 Register ConstZeroReg = 1657 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1658 1659 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1660 .addDef(NotEqualReg) 1661 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1662 .addUse(InputRegister) 1663 .addUse(ConstZeroReg) 1664 .constrainAllUses(TII, TRI, RBI); 1665 } 1666 1667 if (!IsVectorTy) 1668 return Result; 1669 1670 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1671 .addDef(ResVReg) 1672 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1673 .addUse(NotEqualReg) 1674 .constrainAllUses(TII, TRI, RBI); 1675 } 1676 1677 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1678 const SPIRVType *ResType, 1679 MachineInstr &I) const { 1680 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1681 } 1682 1683 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1684 const SPIRVType *ResType, 1685 MachineInstr &I) const { 1686 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1687 } 1688 1689 // Select the OpDot instruction for the given float dot 1690 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1691 const SPIRVType *ResType, 1692 MachineInstr &I) const { 1693 assert(I.getNumOperands() == 4); 1694 assert(I.getOperand(2).isReg()); 1695 assert(I.getOperand(3).isReg()); 1696 1697 [[maybe_unused]] SPIRVType *VecType = 1698 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1699 1700 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1701 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1702 "dot product requires a vector of at least 2 components"); 1703 1704 [[maybe_unused]] SPIRVType *EltType = 1705 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1706 1707 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1708 1709 MachineBasicBlock &BB = *I.getParent(); 1710 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1711 .addDef(ResVReg) 1712 .addUse(GR.getSPIRVTypeID(ResType)) 1713 .addUse(I.getOperand(2).getReg()) 1714 .addUse(I.getOperand(3).getReg()) 1715 .constrainAllUses(TII, TRI, RBI); 1716 } 1717 1718 // Since pre-1.6 SPIRV has no integer dot implementation, 1719 // expand by piecewise multiplying and adding the results 1720 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1721 const SPIRVType *ResType, 1722 MachineInstr &I) const { 1723 assert(I.getNumOperands() == 4); 1724 assert(I.getOperand(2).isReg()); 1725 assert(I.getOperand(3).isReg()); 1726 MachineBasicBlock &BB = *I.getParent(); 1727 1728 // Multiply the vectors, then sum the results 1729 Register Vec0 = I.getOperand(2).getReg(); 1730 Register Vec1 = I.getOperand(3).getReg(); 1731 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1732 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1733 1734 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1735 .addDef(TmpVec) 1736 .addUse(GR.getSPIRVTypeID(VecType)) 1737 .addUse(Vec0) 1738 .addUse(Vec1) 1739 .constrainAllUses(TII, TRI, RBI); 1740 1741 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1742 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1743 "dot product requires a vector of at least 2 components"); 1744 1745 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1746 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1747 .addDef(Res) 1748 .addUse(GR.getSPIRVTypeID(ResType)) 1749 .addUse(TmpVec) 1750 .addImm(0) 1751 .constrainAllUses(TII, TRI, RBI); 1752 1753 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1754 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1755 1756 Result &= 1757 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1758 .addDef(Elt) 1759 .addUse(GR.getSPIRVTypeID(ResType)) 1760 .addUse(TmpVec) 1761 .addImm(i) 1762 .constrainAllUses(TII, TRI, RBI); 1763 1764 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1765 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1766 : ResVReg; 1767 1768 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1769 .addDef(Sum) 1770 .addUse(GR.getSPIRVTypeID(ResType)) 1771 .addUse(Res) 1772 .addUse(Elt) 1773 .constrainAllUses(TII, TRI, RBI); 1774 Res = Sum; 1775 } 1776 1777 return Result; 1778 } 1779 1780 template <bool Signed> 1781 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1782 const SPIRVType *ResType, 1783 MachineInstr &I) const { 1784 assert(I.getNumOperands() == 5); 1785 assert(I.getOperand(2).isReg()); 1786 assert(I.getOperand(3).isReg()); 1787 assert(I.getOperand(4).isReg()); 1788 MachineBasicBlock &BB = *I.getParent(); 1789 1790 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1791 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1792 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1793 .addDef(Dot) 1794 .addUse(GR.getSPIRVTypeID(ResType)) 1795 .addUse(I.getOperand(2).getReg()) 1796 .addUse(I.getOperand(3).getReg()) 1797 .constrainAllUses(TII, TRI, RBI); 1798 1799 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1800 .addDef(ResVReg) 1801 .addUse(GR.getSPIRVTypeID(ResType)) 1802 .addUse(Dot) 1803 .addUse(I.getOperand(4).getReg()) 1804 .constrainAllUses(TII, TRI, RBI); 1805 } 1806 1807 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1808 // extract the elements of the packed inputs, multiply them and add the result 1809 // to the accumulator. 1810 template <bool Signed> 1811 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1812 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1813 assert(I.getNumOperands() == 5); 1814 assert(I.getOperand(2).isReg()); 1815 assert(I.getOperand(3).isReg()); 1816 assert(I.getOperand(4).isReg()); 1817 MachineBasicBlock &BB = *I.getParent(); 1818 1819 bool Result = true; 1820 1821 // Acc = C 1822 Register Acc = I.getOperand(4).getReg(); 1823 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1824 auto ExtractOp = 1825 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1826 1827 // Extract the i8 element, multiply and add it to the accumulator 1828 for (unsigned i = 0; i < 4; i++) { 1829 // A[i] 1830 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1831 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1832 .addDef(AElt) 1833 .addUse(GR.getSPIRVTypeID(ResType)) 1834 .addUse(I.getOperand(2).getReg()) 1835 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1836 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1837 .constrainAllUses(TII, TRI, RBI); 1838 1839 // B[i] 1840 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1841 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1842 .addDef(BElt) 1843 .addUse(GR.getSPIRVTypeID(ResType)) 1844 .addUse(I.getOperand(3).getReg()) 1845 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1846 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1847 .constrainAllUses(TII, TRI, RBI); 1848 1849 // A[i] * B[i] 1850 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1851 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1852 .addDef(Mul) 1853 .addUse(GR.getSPIRVTypeID(ResType)) 1854 .addUse(AElt) 1855 .addUse(BElt) 1856 .constrainAllUses(TII, TRI, RBI); 1857 1858 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1859 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1860 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1861 .addDef(MaskMul) 1862 .addUse(GR.getSPIRVTypeID(ResType)) 1863 .addUse(Mul) 1864 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1865 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1866 .constrainAllUses(TII, TRI, RBI); 1867 1868 // Acc = Acc + A[i] * B[i] 1869 Register Sum = 1870 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1871 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1872 .addDef(Sum) 1873 .addUse(GR.getSPIRVTypeID(ResType)) 1874 .addUse(Acc) 1875 .addUse(MaskMul) 1876 .constrainAllUses(TII, TRI, RBI); 1877 1878 Acc = Sum; 1879 } 1880 1881 return Result; 1882 } 1883 1884 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1885 /// does not have a saturate builtin. 1886 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1887 const SPIRVType *ResType, 1888 MachineInstr &I) const { 1889 assert(I.getNumOperands() == 3); 1890 assert(I.getOperand(2).isReg()); 1891 MachineBasicBlock &BB = *I.getParent(); 1892 Register VZero = buildZerosValF(ResType, I); 1893 Register VOne = buildOnesValF(ResType, I); 1894 1895 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1896 .addDef(ResVReg) 1897 .addUse(GR.getSPIRVTypeID(ResType)) 1898 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1899 .addImm(GL::FClamp) 1900 .addUse(I.getOperand(2).getReg()) 1901 .addUse(VZero) 1902 .addUse(VOne) 1903 .constrainAllUses(TII, TRI, RBI); 1904 } 1905 1906 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1907 const SPIRVType *ResType, 1908 MachineInstr &I) const { 1909 assert(I.getNumOperands() == 3); 1910 assert(I.getOperand(2).isReg()); 1911 MachineBasicBlock &BB = *I.getParent(); 1912 Register InputRegister = I.getOperand(2).getReg(); 1913 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1914 auto &DL = I.getDebugLoc(); 1915 1916 if (!InputType) 1917 report_fatal_error("Input Type could not be determined."); 1918 1919 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1920 1921 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1922 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1923 1924 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1925 1926 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1927 Register SignReg = NeedsConversion 1928 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1929 : ResVReg; 1930 1931 bool Result = 1932 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1933 .addDef(SignReg) 1934 .addUse(GR.getSPIRVTypeID(InputType)) 1935 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1936 .addImm(SignOpcode) 1937 .addUse(InputRegister) 1938 .constrainAllUses(TII, TRI, RBI); 1939 1940 if (NeedsConversion) { 1941 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1942 Result &= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1943 .addDef(ResVReg) 1944 .addUse(GR.getSPIRVTypeID(ResType)) 1945 .addUse(SignReg) 1946 .constrainAllUses(TII, TRI, RBI); 1947 } 1948 1949 return Result; 1950 } 1951 1952 bool SPIRVInstructionSelector::selectWaveActiveCountBits( 1953 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1954 assert(I.getNumOperands() == 3); 1955 assert(I.getOperand(2).isReg()); 1956 MachineBasicBlock &BB = *I.getParent(); 1957 1958 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1959 SPIRVType *BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); 1960 Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); 1961 1962 bool Result = 1963 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformBallot)) 1964 .addDef(BallotReg) 1965 .addUse(GR.getSPIRVTypeID(BallotType)) 1966 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1967 .addUse(I.getOperand(2).getReg()) 1968 .constrainAllUses(TII, TRI, RBI); 1969 1970 Result &= 1971 BuildMI(BB, I, I.getDebugLoc(), 1972 TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) 1973 .addDef(ResVReg) 1974 .addUse(GR.getSPIRVTypeID(ResType)) 1975 .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII)) 1976 .addImm(SPIRV::GroupOperation::Reduce) 1977 .addUse(BallotReg) 1978 .constrainAllUses(TII, TRI, RBI); 1979 1980 return Result; 1981 } 1982 1983 bool SPIRVInstructionSelector::selectWaveReadLaneAt(Register ResVReg, 1984 const SPIRVType *ResType, 1985 MachineInstr &I) const { 1986 assert(I.getNumOperands() == 4); 1987 assert(I.getOperand(2).isReg()); 1988 assert(I.getOperand(3).isReg()); 1989 MachineBasicBlock &BB = *I.getParent(); 1990 1991 // IntTy is used to define the execution scope, set to 3 to denote a 1992 // cross-lane interaction equivalent to a SPIR-V subgroup. 1993 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1994 return BuildMI(BB, I, I.getDebugLoc(), 1995 TII.get(SPIRV::OpGroupNonUniformShuffle)) 1996 .addDef(ResVReg) 1997 .addUse(GR.getSPIRVTypeID(ResType)) 1998 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)) 1999 .addUse(I.getOperand(2).getReg()) 2000 .addUse(I.getOperand(3).getReg()); 2001 } 2002 2003 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 2004 const SPIRVType *ResType, 2005 MachineInstr &I) const { 2006 MachineBasicBlock &BB = *I.getParent(); 2007 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 2008 .addDef(ResVReg) 2009 .addUse(GR.getSPIRVTypeID(ResType)) 2010 .addUse(I.getOperand(1).getReg()) 2011 .constrainAllUses(TII, TRI, RBI); 2012 } 2013 2014 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 2015 const SPIRVType *ResType, 2016 MachineInstr &I) const { 2017 // There is no way to implement `freeze` correctly without support on SPIR-V 2018 // standard side, but we may at least address a simple (static) case when 2019 // undef/poison value presence is obvious. The main benefit of even 2020 // incomplete `freeze` support is preventing of translation from crashing due 2021 // to lack of support on legalization and instruction selection steps. 2022 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 2023 return false; 2024 Register OpReg = I.getOperand(1).getReg(); 2025 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 2026 Register Reg; 2027 switch (Def->getOpcode()) { 2028 case SPIRV::ASSIGN_TYPE: 2029 if (MachineInstr *AssignToDef = 2030 MRI->getVRegDef(Def->getOperand(1).getReg())) { 2031 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 2032 Reg = Def->getOperand(2).getReg(); 2033 } 2034 break; 2035 case SPIRV::OpUndef: 2036 Reg = Def->getOperand(1).getReg(); 2037 break; 2038 } 2039 unsigned DestOpCode; 2040 if (Reg.isValid()) { 2041 DestOpCode = SPIRV::OpConstantNull; 2042 } else { 2043 DestOpCode = TargetOpcode::COPY; 2044 Reg = OpReg; 2045 } 2046 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 2047 .addDef(I.getOperand(0).getReg()) 2048 .addUse(Reg) 2049 .constrainAllUses(TII, TRI, RBI); 2050 } 2051 return false; 2052 } 2053 2054 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 2055 const SPIRVType *ResType) { 2056 Register OpReg = ResType->getOperand(2).getReg(); 2057 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 2058 if (!OpDef) 2059 return 0; 2060 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2061 OpDef->getOperand(1).isReg()) { 2062 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2063 OpDef = RefDef; 2064 } 2065 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 2066 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 2067 : 0; 2068 return N; 2069 } 2070 2071 // Return true if the type represents a constant register 2072 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 2073 SmallPtrSet<SPIRVType *, 4> &Visited) { 2074 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 2075 OpDef->getOperand(1).isReg()) { 2076 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 2077 OpDef = RefDef; 2078 } 2079 2080 if (Visited.contains(OpDef)) 2081 return true; 2082 Visited.insert(OpDef); 2083 2084 unsigned Opcode = OpDef->getOpcode(); 2085 switch (Opcode) { 2086 case TargetOpcode::G_CONSTANT: 2087 case TargetOpcode::G_FCONSTANT: 2088 return true; 2089 case TargetOpcode::G_INTRINSIC: 2090 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 2091 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 2092 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 2093 Intrinsic::spv_const_composite; 2094 case TargetOpcode::G_BUILD_VECTOR: 2095 case TargetOpcode::G_SPLAT_VECTOR: { 2096 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 2097 i++) { 2098 SPIRVType *OpNestedDef = 2099 OpDef->getOperand(i).isReg() 2100 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 2101 : nullptr; 2102 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 2103 return false; 2104 } 2105 return true; 2106 } 2107 } 2108 return false; 2109 } 2110 2111 // Return true if the virtual register represents a constant 2112 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2113 SmallPtrSet<SPIRVType *, 4> Visited; 2114 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2115 return isConstReg(MRI, OpDef, Visited); 2116 return false; 2117 } 2118 2119 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2120 const SPIRVType *ResType, 2121 MachineInstr &I) const { 2122 unsigned N = 0; 2123 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2124 N = GR.getScalarOrVectorComponentCount(ResType); 2125 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2126 N = getArrayComponentCount(MRI, ResType); 2127 else 2128 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2129 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2130 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2131 2132 // check if we may construct a constant vector 2133 bool IsConst = true; 2134 for (unsigned i = I.getNumExplicitDefs(); 2135 i < I.getNumExplicitOperands() && IsConst; ++i) 2136 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2137 IsConst = false; 2138 2139 if (!IsConst && N < 2) 2140 report_fatal_error( 2141 "There must be at least two constituent operands in a vector"); 2142 2143 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2144 TII.get(IsConst ? SPIRV::OpConstantComposite 2145 : SPIRV::OpCompositeConstruct)) 2146 .addDef(ResVReg) 2147 .addUse(GR.getSPIRVTypeID(ResType)); 2148 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2149 MIB.addUse(I.getOperand(i).getReg()); 2150 return MIB.constrainAllUses(TII, TRI, RBI); 2151 } 2152 2153 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2154 const SPIRVType *ResType, 2155 MachineInstr &I) const { 2156 unsigned N = 0; 2157 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2158 N = GR.getScalarOrVectorComponentCount(ResType); 2159 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2160 N = getArrayComponentCount(MRI, ResType); 2161 else 2162 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2163 2164 unsigned OpIdx = I.getNumExplicitDefs(); 2165 if (!I.getOperand(OpIdx).isReg()) 2166 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2167 2168 // check if we may construct a constant vector 2169 Register OpReg = I.getOperand(OpIdx).getReg(); 2170 bool IsConst = isConstReg(MRI, OpReg); 2171 2172 if (!IsConst && N < 2) 2173 report_fatal_error( 2174 "There must be at least two constituent operands in a vector"); 2175 2176 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2177 TII.get(IsConst ? SPIRV::OpConstantComposite 2178 : SPIRV::OpCompositeConstruct)) 2179 .addDef(ResVReg) 2180 .addUse(GR.getSPIRVTypeID(ResType)); 2181 for (unsigned i = 0; i < N; ++i) 2182 MIB.addUse(OpReg); 2183 return MIB.constrainAllUses(TII, TRI, RBI); 2184 } 2185 2186 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2187 const SPIRVType *ResType, 2188 unsigned CmpOpc, 2189 MachineInstr &I) const { 2190 Register Cmp0 = I.getOperand(2).getReg(); 2191 Register Cmp1 = I.getOperand(3).getReg(); 2192 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2193 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2194 "CMP operands should have the same type"); 2195 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2196 .addDef(ResVReg) 2197 .addUse(GR.getSPIRVTypeID(ResType)) 2198 .addUse(Cmp0) 2199 .addUse(Cmp1) 2200 .constrainAllUses(TII, TRI, RBI); 2201 } 2202 2203 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2204 const SPIRVType *ResType, 2205 MachineInstr &I) const { 2206 auto Pred = I.getOperand(1).getPredicate(); 2207 unsigned CmpOpc; 2208 2209 Register CmpOperand = I.getOperand(2).getReg(); 2210 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2211 CmpOpc = getPtrCmpOpcode(Pred); 2212 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2213 CmpOpc = getBoolCmpOpcode(Pred); 2214 else 2215 CmpOpc = getICmpOpcode(Pred); 2216 return selectCmp(ResVReg, ResType, CmpOpc, I); 2217 } 2218 2219 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2220 const MachineInstr &I, 2221 int OpIdx) const { 2222 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2223 "Expected G_FCONSTANT"); 2224 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2225 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2226 } 2227 2228 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2229 const MachineInstr &I, 2230 int OpIdx) const { 2231 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2232 "Expected G_CONSTANT"); 2233 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2234 } 2235 2236 std::pair<Register, bool> 2237 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2238 const SPIRVType *ResType) const { 2239 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2240 const SPIRVType *SpvI32Ty = 2241 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2242 // Find a constant in DT or build a new one. 2243 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2244 Register NewReg = GR.find(ConstInt, GR.CurMF); 2245 bool Result = true; 2246 if (!NewReg.isValid()) { 2247 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2248 GR.add(ConstInt, GR.CurMF, NewReg); 2249 MachineInstr *MI; 2250 MachineBasicBlock &BB = *I.getParent(); 2251 if (Val == 0) { 2252 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2253 .addDef(NewReg) 2254 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2255 } else { 2256 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2257 .addDef(NewReg) 2258 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2259 .addImm(APInt(32, Val).getZExtValue()); 2260 } 2261 Result &= constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2262 } 2263 return {NewReg, Result}; 2264 } 2265 2266 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2267 const SPIRVType *ResType, 2268 MachineInstr &I) const { 2269 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2270 return selectCmp(ResVReg, ResType, CmpOp, I); 2271 } 2272 2273 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2274 MachineInstr &I) const { 2275 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2276 bool ZeroAsNull = STI.isOpenCLEnv(); 2277 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2278 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2279 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2280 } 2281 2282 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2283 MachineInstr &I) const { 2284 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2285 bool ZeroAsNull = STI.isOpenCLEnv(); 2286 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2287 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2288 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2289 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2290 } 2291 2292 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2293 MachineInstr &I) const { 2294 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2295 bool ZeroAsNull = STI.isOpenCLEnv(); 2296 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2297 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2298 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2299 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2300 } 2301 2302 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2303 const SPIRVType *ResType, 2304 MachineInstr &I) const { 2305 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2306 APInt One = 2307 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2308 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2309 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2310 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2311 } 2312 2313 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2314 const SPIRVType *ResType, 2315 MachineInstr &I, 2316 bool IsSigned) const { 2317 // To extend a bool, we need to use OpSelect between constants. 2318 Register ZeroReg = buildZerosVal(ResType, I); 2319 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2320 bool IsScalarBool = 2321 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2322 unsigned Opcode = 2323 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2324 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2325 .addDef(ResVReg) 2326 .addUse(GR.getSPIRVTypeID(ResType)) 2327 .addUse(I.getOperand(1).getReg()) 2328 .addUse(OneReg) 2329 .addUse(ZeroReg) 2330 .constrainAllUses(TII, TRI, RBI); 2331 } 2332 2333 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2334 const SPIRVType *ResType, 2335 MachineInstr &I, bool IsSigned, 2336 unsigned Opcode) const { 2337 Register SrcReg = I.getOperand(1).getReg(); 2338 // We can convert bool value directly to float type without OpConvert*ToF, 2339 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2340 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2341 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2342 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2343 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2344 const unsigned NumElts = ResType->getOperand(2).getImm(); 2345 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2346 } 2347 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2348 selectSelect(SrcReg, TmpType, I, false); 2349 } 2350 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2351 } 2352 2353 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2354 const SPIRVType *ResType, 2355 MachineInstr &I, bool IsSigned) const { 2356 Register SrcReg = I.getOperand(1).getReg(); 2357 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2358 return selectSelect(ResVReg, ResType, I, IsSigned); 2359 2360 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2361 if (SrcType == ResType) { 2362 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2363 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2364 if (DstRC != SrcRC && SrcRC) 2365 MRI->setRegClass(ResVReg, SrcRC); 2366 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2367 TII.get(TargetOpcode::COPY)) 2368 .addDef(ResVReg) 2369 .addUse(SrcReg) 2370 .constrainAllUses(TII, TRI, RBI); 2371 } 2372 2373 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2374 return selectUnOp(ResVReg, ResType, I, Opcode); 2375 } 2376 2377 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2378 Register ResVReg, 2379 MachineInstr &I, 2380 const SPIRVType *IntTy, 2381 const SPIRVType *BoolTy) const { 2382 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2383 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2384 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2385 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2386 Register Zero = buildZerosVal(IntTy, I); 2387 Register One = buildOnesVal(false, IntTy, I); 2388 MachineBasicBlock &BB = *I.getParent(); 2389 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2390 .addDef(BitIntReg) 2391 .addUse(GR.getSPIRVTypeID(IntTy)) 2392 .addUse(IntReg) 2393 .addUse(One) 2394 .constrainAllUses(TII, TRI, RBI); 2395 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2396 .addDef(ResVReg) 2397 .addUse(GR.getSPIRVTypeID(BoolTy)) 2398 .addUse(BitIntReg) 2399 .addUse(Zero) 2400 .constrainAllUses(TII, TRI, RBI); 2401 } 2402 2403 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2404 const SPIRVType *ResType, 2405 MachineInstr &I) const { 2406 Register IntReg = I.getOperand(1).getReg(); 2407 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2408 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2409 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2410 if (ArgType == ResType) { 2411 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2412 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2413 if (DstRC != SrcRC && SrcRC) 2414 MRI->setRegClass(ResVReg, SrcRC); 2415 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2416 TII.get(TargetOpcode::COPY)) 2417 .addDef(ResVReg) 2418 .addUse(IntReg) 2419 .constrainAllUses(TII, TRI, RBI); 2420 } 2421 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2422 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2423 return selectUnOp(ResVReg, ResType, I, Opcode); 2424 } 2425 2426 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2427 const SPIRVType *ResType, 2428 const APInt &Imm, 2429 MachineInstr &I) const { 2430 unsigned TyOpcode = ResType->getOpcode(); 2431 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2432 MachineBasicBlock &BB = *I.getParent(); 2433 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2434 Imm.isZero()) 2435 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2436 .addDef(ResVReg) 2437 .addUse(GR.getSPIRVTypeID(ResType)) 2438 .constrainAllUses(TII, TRI, RBI); 2439 if (TyOpcode == SPIRV::OpTypeInt) { 2440 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2441 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2442 if (Reg == ResVReg) 2443 return true; 2444 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2445 .addDef(ResVReg) 2446 .addUse(Reg) 2447 .constrainAllUses(TII, TRI, RBI); 2448 } 2449 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2450 .addDef(ResVReg) 2451 .addUse(GR.getSPIRVTypeID(ResType)); 2452 // <=32-bit integers should be caught by the sdag pattern. 2453 assert(Imm.getBitWidth() > 32); 2454 addNumImm(Imm, MIB); 2455 return MIB.constrainAllUses(TII, TRI, RBI); 2456 } 2457 2458 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2459 const SPIRVType *ResType, 2460 MachineInstr &I) const { 2461 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2462 .addDef(ResVReg) 2463 .addUse(GR.getSPIRVTypeID(ResType)) 2464 .constrainAllUses(TII, TRI, RBI); 2465 } 2466 2467 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2468 assert(MO.isReg()); 2469 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2470 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2471 assert(TypeInst->getOperand(1).isReg()); 2472 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2473 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2474 } 2475 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2476 } 2477 2478 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2479 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2480 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2481 return TypeInst->getOperand(2).getImm(); 2482 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2483 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2484 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2485 } 2486 2487 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2488 const SPIRVType *ResType, 2489 MachineInstr &I) const { 2490 MachineBasicBlock &BB = *I.getParent(); 2491 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2492 .addDef(ResVReg) 2493 .addUse(GR.getSPIRVTypeID(ResType)) 2494 // object to insert 2495 .addUse(I.getOperand(3).getReg()) 2496 // composite to insert into 2497 .addUse(I.getOperand(2).getReg()); 2498 for (unsigned i = 4; i < I.getNumOperands(); i++) 2499 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2500 return MIB.constrainAllUses(TII, TRI, RBI); 2501 } 2502 2503 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2504 const SPIRVType *ResType, 2505 MachineInstr &I) const { 2506 MachineBasicBlock &BB = *I.getParent(); 2507 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2508 .addDef(ResVReg) 2509 .addUse(GR.getSPIRVTypeID(ResType)) 2510 .addUse(I.getOperand(2).getReg()); 2511 for (unsigned i = 3; i < I.getNumOperands(); i++) 2512 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2513 return MIB.constrainAllUses(TII, TRI, RBI); 2514 } 2515 2516 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2517 const SPIRVType *ResType, 2518 MachineInstr &I) const { 2519 if (isImm(I.getOperand(4), MRI)) 2520 return selectInsertVal(ResVReg, ResType, I); 2521 MachineBasicBlock &BB = *I.getParent(); 2522 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2523 .addDef(ResVReg) 2524 .addUse(GR.getSPIRVTypeID(ResType)) 2525 .addUse(I.getOperand(2).getReg()) 2526 .addUse(I.getOperand(3).getReg()) 2527 .addUse(I.getOperand(4).getReg()) 2528 .constrainAllUses(TII, TRI, RBI); 2529 } 2530 2531 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2532 const SPIRVType *ResType, 2533 MachineInstr &I) const { 2534 if (isImm(I.getOperand(3), MRI)) 2535 return selectExtractVal(ResVReg, ResType, I); 2536 MachineBasicBlock &BB = *I.getParent(); 2537 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2538 .addDef(ResVReg) 2539 .addUse(GR.getSPIRVTypeID(ResType)) 2540 .addUse(I.getOperand(2).getReg()) 2541 .addUse(I.getOperand(3).getReg()) 2542 .constrainAllUses(TII, TRI, RBI); 2543 } 2544 2545 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2546 const SPIRVType *ResType, 2547 MachineInstr &I) const { 2548 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2549 2550 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2551 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2552 // we have to use Op[InBounds]AccessChain. 2553 const unsigned Opcode = STI.isVulkanEnv() 2554 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2555 : SPIRV::OpAccessChain) 2556 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2557 : SPIRV::OpPtrAccessChain); 2558 2559 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2560 .addDef(ResVReg) 2561 .addUse(GR.getSPIRVTypeID(ResType)) 2562 // Object to get a pointer to. 2563 .addUse(I.getOperand(3).getReg()); 2564 // Adding indices. 2565 const unsigned StartingIndex = 2566 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2567 ? 5 2568 : 4; 2569 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2570 Res.addUse(I.getOperand(i).getReg()); 2571 return Res.constrainAllUses(TII, TRI, RBI); 2572 } 2573 2574 // Maybe wrap a value into OpSpecConstantOp 2575 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2576 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2577 bool Result = true; 2578 unsigned Lim = I.getNumExplicitOperands(); 2579 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2580 Register OpReg = I.getOperand(i).getReg(); 2581 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2582 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2583 SmallPtrSet<SPIRVType *, 4> Visited; 2584 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2585 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2586 GR.isAggregateType(OpType)) { 2587 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2588 // by selectAddrSpaceCast() 2589 CompositeArgs.push_back(OpReg); 2590 continue; 2591 } 2592 MachineFunction *MF = I.getMF(); 2593 Register WrapReg = GR.find(OpDefine, MF); 2594 if (WrapReg.isValid()) { 2595 CompositeArgs.push_back(WrapReg); 2596 continue; 2597 } 2598 // Create a new register for the wrapper 2599 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2600 GR.add(OpDefine, MF, WrapReg); 2601 CompositeArgs.push_back(WrapReg); 2602 // Decorate the wrapper register and generate a new instruction 2603 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2604 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2605 MachineBasicBlock &BB = *I.getParent(); 2606 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2607 .addDef(WrapReg) 2608 .addUse(GR.getSPIRVTypeID(OpType)) 2609 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2610 .addUse(OpReg) 2611 .constrainAllUses(TII, TRI, RBI); 2612 if (!Result) 2613 break; 2614 } 2615 return Result; 2616 } 2617 2618 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2619 const SPIRVType *ResType, 2620 MachineInstr &I) const { 2621 MachineBasicBlock &BB = *I.getParent(); 2622 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2623 switch (IID) { 2624 case Intrinsic::spv_load: 2625 return selectLoad(ResVReg, ResType, I); 2626 case Intrinsic::spv_store: 2627 return selectStore(I); 2628 case Intrinsic::spv_extractv: 2629 return selectExtractVal(ResVReg, ResType, I); 2630 case Intrinsic::spv_insertv: 2631 return selectInsertVal(ResVReg, ResType, I); 2632 case Intrinsic::spv_extractelt: 2633 return selectExtractElt(ResVReg, ResType, I); 2634 case Intrinsic::spv_insertelt: 2635 return selectInsertElt(ResVReg, ResType, I); 2636 case Intrinsic::spv_gep: 2637 return selectGEP(ResVReg, ResType, I); 2638 case Intrinsic::spv_unref_global: 2639 case Intrinsic::spv_init_global: { 2640 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2641 MachineInstr *Init = I.getNumExplicitOperands() > 2 2642 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2643 : nullptr; 2644 assert(MI); 2645 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2646 } 2647 case Intrinsic::spv_undef: { 2648 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2649 .addDef(ResVReg) 2650 .addUse(GR.getSPIRVTypeID(ResType)); 2651 return MIB.constrainAllUses(TII, TRI, RBI); 2652 } 2653 case Intrinsic::spv_const_composite: { 2654 // If no values are attached, the composite is null constant. 2655 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2656 // Select a proper instruction. 2657 unsigned Opcode = SPIRV::OpConstantNull; 2658 SmallVector<Register> CompositeArgs; 2659 if (!IsNull) { 2660 Opcode = SPIRV::OpConstantComposite; 2661 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2662 return false; 2663 } 2664 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2665 .addDef(ResVReg) 2666 .addUse(GR.getSPIRVTypeID(ResType)); 2667 // skip type MD node we already used when generated assign.type for this 2668 if (!IsNull) { 2669 for (Register OpReg : CompositeArgs) 2670 MIB.addUse(OpReg); 2671 } 2672 return MIB.constrainAllUses(TII, TRI, RBI); 2673 } 2674 case Intrinsic::spv_assign_name: { 2675 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2676 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2677 for (unsigned i = I.getNumExplicitDefs() + 2; 2678 i < I.getNumExplicitOperands(); ++i) { 2679 MIB.addImm(I.getOperand(i).getImm()); 2680 } 2681 return MIB.constrainAllUses(TII, TRI, RBI); 2682 } 2683 case Intrinsic::spv_switch: { 2684 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2685 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2686 if (I.getOperand(i).isReg()) 2687 MIB.addReg(I.getOperand(i).getReg()); 2688 else if (I.getOperand(i).isCImm()) 2689 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2690 else if (I.getOperand(i).isMBB()) 2691 MIB.addMBB(I.getOperand(i).getMBB()); 2692 else 2693 llvm_unreachable("Unexpected OpSwitch operand"); 2694 } 2695 return MIB.constrainAllUses(TII, TRI, RBI); 2696 } 2697 case Intrinsic::spv_loop_merge: 2698 case Intrinsic::spv_selection_merge: { 2699 const auto Opcode = IID == Intrinsic::spv_selection_merge 2700 ? SPIRV::OpSelectionMerge 2701 : SPIRV::OpLoopMerge; 2702 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2703 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2704 assert(I.getOperand(i).isMBB()); 2705 MIB.addMBB(I.getOperand(i).getMBB()); 2706 } 2707 MIB.addImm(SPIRV::SelectionControl::None); 2708 return MIB.constrainAllUses(TII, TRI, RBI); 2709 } 2710 case Intrinsic::spv_cmpxchg: 2711 return selectAtomicCmpXchg(ResVReg, ResType, I); 2712 case Intrinsic::spv_unreachable: 2713 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) 2714 .constrainAllUses(TII, TRI, RBI); 2715 case Intrinsic::spv_alloca: 2716 return selectFrameIndex(ResVReg, ResType, I); 2717 case Intrinsic::spv_alloca_array: 2718 return selectAllocaArray(ResVReg, ResType, I); 2719 case Intrinsic::spv_assume: 2720 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2721 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2722 .addUse(I.getOperand(1).getReg()) 2723 .constrainAllUses(TII, TRI, RBI); 2724 break; 2725 case Intrinsic::spv_expect: 2726 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2727 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2728 .addDef(ResVReg) 2729 .addUse(GR.getSPIRVTypeID(ResType)) 2730 .addUse(I.getOperand(2).getReg()) 2731 .addUse(I.getOperand(3).getReg()) 2732 .constrainAllUses(TII, TRI, RBI); 2733 break; 2734 case Intrinsic::arithmetic_fence: 2735 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2736 return BuildMI(BB, I, I.getDebugLoc(), 2737 TII.get(SPIRV::OpArithmeticFenceEXT)) 2738 .addDef(ResVReg) 2739 .addUse(GR.getSPIRVTypeID(ResType)) 2740 .addUse(I.getOperand(2).getReg()) 2741 .constrainAllUses(TII, TRI, RBI); 2742 else 2743 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), 2744 ResVReg) 2745 .addUse(I.getOperand(2).getReg()) 2746 .constrainAllUses(TII, TRI, RBI); 2747 break; 2748 case Intrinsic::spv_thread_id: 2749 return selectSpvThreadId(ResVReg, ResType, I); 2750 case Intrinsic::spv_fdot: 2751 return selectFloatDot(ResVReg, ResType, I); 2752 case Intrinsic::spv_udot: 2753 case Intrinsic::spv_sdot: 2754 return selectIntegerDot(ResVReg, ResType, I); 2755 case Intrinsic::spv_dot4add_i8packed: 2756 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2757 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2758 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2759 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2760 case Intrinsic::spv_dot4add_u8packed: 2761 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2762 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2763 return selectDot4AddPacked<false>(ResVReg, ResType, I); 2764 return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); 2765 case Intrinsic::spv_all: 2766 return selectAll(ResVReg, ResType, I); 2767 case Intrinsic::spv_any: 2768 return selectAny(ResVReg, ResType, I); 2769 case Intrinsic::spv_cross: 2770 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2771 case Intrinsic::spv_lerp: 2772 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2773 case Intrinsic::spv_length: 2774 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2775 case Intrinsic::spv_degrees: 2776 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2777 case Intrinsic::spv_frac: 2778 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2779 case Intrinsic::spv_normalize: 2780 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2781 case Intrinsic::spv_rsqrt: 2782 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2783 case Intrinsic::spv_sign: 2784 return selectSign(ResVReg, ResType, I); 2785 case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb 2786 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); 2787 case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb 2788 return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); 2789 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2790 bool Result = true; 2791 auto MemSemConstant = 2792 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2793 Register MemSemReg = MemSemConstant.first; 2794 Result &= MemSemConstant.second; 2795 auto ScopeConstant = buildI32Constant(SPIRV::Scope::Workgroup, I); 2796 Register ScopeReg = ScopeConstant.first; 2797 Result &= ScopeConstant.second; 2798 MachineBasicBlock &BB = *I.getParent(); 2799 return Result && 2800 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2801 .addUse(ScopeReg) 2802 .addUse(ScopeReg) 2803 .addUse(MemSemReg) 2804 .constrainAllUses(TII, TRI, RBI); 2805 } 2806 case Intrinsic::spv_lifetime_start: 2807 case Intrinsic::spv_lifetime_end: { 2808 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2809 : SPIRV::OpLifetimeStop; 2810 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2811 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2812 if (Size == -1) 2813 Size = 0; 2814 return BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) 2815 .addUse(PtrReg) 2816 .addImm(Size) 2817 .constrainAllUses(TII, TRI, RBI); 2818 } 2819 case Intrinsic::spv_saturate: 2820 return selectSaturate(ResVReg, ResType, I); 2821 case Intrinsic::spv_nclamp: 2822 return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); 2823 case Intrinsic::spv_uclamp: 2824 return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); 2825 case Intrinsic::spv_sclamp: 2826 return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); 2827 case Intrinsic::spv_wave_active_countbits: 2828 return selectWaveActiveCountBits(ResVReg, ResType, I); 2829 case Intrinsic::spv_wave_is_first_lane: { 2830 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2831 return BuildMI(BB, I, I.getDebugLoc(), 2832 TII.get(SPIRV::OpGroupNonUniformElect)) 2833 .addDef(ResVReg) 2834 .addUse(GR.getSPIRVTypeID(ResType)) 2835 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2836 } 2837 case Intrinsic::spv_wave_readlane: 2838 return selectWaveReadLaneAt(ResVReg, ResType, I); 2839 case Intrinsic::spv_step: 2840 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2841 case Intrinsic::spv_radians: 2842 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2843 // Discard intrinsics which we do not expect to actually represent code after 2844 // lowering or intrinsics which are not implemented but should not crash when 2845 // found in a customer's LLVM IR input. 2846 case Intrinsic::instrprof_increment: 2847 case Intrinsic::instrprof_increment_step: 2848 case Intrinsic::instrprof_value_profile: 2849 break; 2850 // Discard internal intrinsics. 2851 case Intrinsic::spv_value_md: 2852 break; 2853 case Intrinsic::spv_handle_fromBinding: { 2854 return selectHandleFromBinding(ResVReg, ResType, I); 2855 } 2856 case Intrinsic::spv_typedBufferLoad: { 2857 selectReadImageIntrinsic(ResVReg, ResType, I); 2858 return true; 2859 } 2860 default: { 2861 std::string DiagMsg; 2862 raw_string_ostream OS(DiagMsg); 2863 I.print(OS); 2864 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2865 report_fatal_error(DiagMsg.c_str(), false); 2866 } 2867 } 2868 return true; 2869 } 2870 2871 bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2872 const SPIRVType *ResType, 2873 MachineInstr &I) const { 2874 2875 uint32_t Set = foldImm(I.getOperand(2), MRI); 2876 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2877 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2878 Register IndexReg = I.getOperand(5).getReg(); 2879 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2880 2881 MachineIRBuilder MIRBuilder(I); 2882 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2883 IndexReg, IsNonUniform, MIRBuilder); 2884 2885 if (IsNonUniform) 2886 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2887 2888 // TODO: For now we assume the resource is an image, which needs to be 2889 // loaded to get the handle. That will not be true for storage buffers. 2890 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2891 .addDef(ResVReg) 2892 .addUse(GR.getSPIRVTypeID(ResType)) 2893 .addUse(VarReg) 2894 .constrainAllUses(TII, TRI, RBI); 2895 } 2896 2897 void SPIRVInstructionSelector::selectReadImageIntrinsic( 2898 Register &ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 2899 2900 // If the load of the image is in a different basic block, then 2901 // this will generate invalid code. A proper solution is to move 2902 // the OpLoad from selectHandleFromBinding here. However, to do 2903 // that we will need to change the return type of the intrinsic. 2904 // We will do that when we can, but for now trying to move forward with other 2905 // issues. 2906 Register ImageReg = I.getOperand(2).getReg(); 2907 assert(MRI->getVRegDef(ImageReg)->getParent() == I.getParent() && 2908 "The image must be loaded in the same basic block as its use."); 2909 2910 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2911 if (ResultSize == 4) { 2912 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 2913 .addDef(ResVReg) 2914 .addUse(GR.getSPIRVTypeID(ResType)) 2915 .addUse(ImageReg) 2916 .addUse(I.getOperand(3).getReg()); 2917 return; 2918 } 2919 2920 SPIRVType *ReadType = widenTypeToVec4(ResType, I); 2921 Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType)); 2922 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageRead)) 2923 .addDef(ReadReg) 2924 .addUse(GR.getSPIRVTypeID(ReadType)) 2925 .addUse(ImageReg) 2926 .addUse(I.getOperand(3).getReg()); 2927 2928 if (ResultSize == 1) { 2929 BuildMI(*I.getParent(), I, I.getDebugLoc(), 2930 TII.get(SPIRV::OpCompositeExtract)) 2931 .addDef(ResVReg) 2932 .addUse(GR.getSPIRVTypeID(ResType)) 2933 .addUse(ReadReg) 2934 .addImm(0); 2935 return; 2936 } 2937 extractSubvector(ResVReg, ResType, ReadReg, I); 2938 } 2939 2940 void SPIRVInstructionSelector::extractSubvector( 2941 Register &ResVReg, const SPIRVType *ResType, Register &ReadReg, 2942 MachineInstr &InsertionPoint) const { 2943 SPIRVType *InputType = GR.getResultType(ReadReg); 2944 uint64_t InputSize = GR.getScalarOrVectorComponentCount(InputType); 2945 uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); 2946 assert(InputSize > 1 && "The input must be a vector."); 2947 assert(ResultSize > 1 && "The result must be a vector."); 2948 assert(ResultSize < InputSize && 2949 "Cannot extract more element than there are in the input."); 2950 SmallVector<Register> ComponentRegisters; 2951 SPIRVType *ScalarType = GR.getScalarOrVectorComponentType(ResType); 2952 const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); 2953 for (uint64_t I = 0; I < ResultSize; I++) { 2954 Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); 2955 BuildMI(*InsertionPoint.getParent(), InsertionPoint, 2956 InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2957 .addDef(ComponentReg) 2958 .addUse(ScalarType->getOperand(0).getReg()) 2959 .addUse(ReadReg) 2960 .addImm(I); 2961 ComponentRegisters.emplace_back(ComponentReg); 2962 } 2963 2964 MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, 2965 InsertionPoint.getDebugLoc(), 2966 TII.get(SPIRV::OpCompositeConstruct)) 2967 .addDef(ResVReg) 2968 .addUse(GR.getSPIRVTypeID(ResType)); 2969 2970 for (Register ComponentReg : ComponentRegisters) 2971 MIB.addUse(ComponentReg); 2972 } 2973 2974 Register SPIRVInstructionSelector::buildPointerToResource( 2975 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 2976 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 2977 MachineIRBuilder MIRBuilder) const { 2978 if (ArraySize == 1) 2979 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 2980 MIRBuilder); 2981 2982 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 2983 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 2984 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 2985 VarType, Set, Binding, MIRBuilder); 2986 2987 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 2988 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 2989 2990 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2991 if (IsNonUniform) { 2992 // It is unclear which value needs to be marked an non-uniform, so both 2993 // the index and the access changed are decorated as non-uniform. 2994 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2995 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2996 } 2997 2998 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 2999 .addDef(AcReg) 3000 .addUse(GR.getSPIRVTypeID(ResPointerType)) 3001 .addUse(VarReg) 3002 .addUse(IndexReg); 3003 3004 return AcReg; 3005 } 3006 3007 bool SPIRVInstructionSelector::selectFirstBitHigh16(Register ResVReg, 3008 const SPIRVType *ResType, 3009 MachineInstr &I, 3010 bool IsSigned) const { 3011 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 3012 // zero or sign extend 3013 Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3014 bool Result = 3015 selectUnOpWithSrc(ExtReg, ResType, I, I.getOperand(2).getReg(), Opcode); 3016 return Result && selectFirstBitHigh32(ResVReg, ResType, I, ExtReg, IsSigned); 3017 } 3018 3019 bool SPIRVInstructionSelector::selectFirstBitHigh32(Register ResVReg, 3020 const SPIRVType *ResType, 3021 MachineInstr &I, 3022 Register SrcReg, 3023 bool IsSigned) const { 3024 unsigned Opcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; 3025 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3026 .addDef(ResVReg) 3027 .addUse(GR.getSPIRVTypeID(ResType)) 3028 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3029 .addImm(Opcode) 3030 .addUse(SrcReg) 3031 .constrainAllUses(TII, TRI, RBI); 3032 } 3033 3034 bool SPIRVInstructionSelector::selectFirstBitHigh64(Register ResVReg, 3035 const SPIRVType *ResType, 3036 MachineInstr &I, 3037 bool IsSigned) const { 3038 Register OpReg = I.getOperand(2).getReg(); 3039 // 1. split our int64 into 2 pieces using a bitcast 3040 unsigned count = GR.getScalarOrVectorComponentCount(ResType); 3041 SPIRVType *baseType = GR.retrieveScalarOrVectorIntType(ResType); 3042 MachineIRBuilder MIRBuilder(I); 3043 SPIRVType *postCastT = 3044 GR.getOrCreateSPIRVVectorType(baseType, 2 * count, MIRBuilder); 3045 Register bitcastReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3046 bool Result = 3047 selectUnOpWithSrc(bitcastReg, postCastT, I, OpReg, SPIRV::OpBitcast); 3048 3049 // 2. call firstbithigh 3050 Register FBHReg = MRI->createVirtualRegister(GR.getRegClass(postCastT)); 3051 Result &= selectFirstBitHigh32(FBHReg, postCastT, I, bitcastReg, IsSigned); 3052 3053 // 3. split result vector into high bits and low bits 3054 Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3055 Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3056 3057 bool ZeroAsNull = STI.isOpenCLEnv(); 3058 bool isScalarRes = ResType->getOpcode() != SPIRV::OpTypeVector; 3059 if (isScalarRes) { 3060 // if scalar do a vector extract 3061 Result &= selectNAryOpWithSrcs( 3062 HighReg, ResType, I, 3063 {FBHReg, GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull)}, 3064 SPIRV::OpVectorExtractDynamic); 3065 Result &= selectNAryOpWithSrcs( 3066 LowReg, ResType, I, 3067 {FBHReg, GR.getOrCreateConstInt(1, I, ResType, TII, ZeroAsNull)}, 3068 SPIRV::OpVectorExtractDynamic); 3069 } else { // vector case do a shufflevector 3070 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3071 TII.get(SPIRV::OpVectorShuffle)) 3072 .addDef(HighReg) 3073 .addUse(GR.getSPIRVTypeID(ResType)) 3074 .addUse(FBHReg) 3075 .addUse(FBHReg); 3076 // ^^ this vector will not be selected from; could be empty 3077 unsigned j; 3078 for (j = 0; j < count * 2; j += 2) { 3079 MIB.addImm(j); 3080 } 3081 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3082 3083 // get low bits 3084 MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 3085 TII.get(SPIRV::OpVectorShuffle)) 3086 .addDef(LowReg) 3087 .addUse(GR.getSPIRVTypeID(ResType)) 3088 .addUse(FBHReg) 3089 .addUse(FBHReg); 3090 // ^^ this vector will not be selected from; could be empty 3091 for (j = 1; j < count * 2; j += 2) { 3092 MIB.addImm(j); 3093 } 3094 Result &= MIB.constrainAllUses(TII, TRI, RBI); 3095 } 3096 3097 // 4. check if result of each top 32 bits is == -1 3098 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 3099 Register NegOneReg; 3100 Register Reg0; 3101 Register Reg32; 3102 unsigned selectOp; 3103 unsigned addOp; 3104 if (isScalarRes) { 3105 NegOneReg = 3106 GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); 3107 Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 3108 Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); 3109 selectOp = SPIRV::OpSelectSISCond; 3110 addOp = SPIRV::OpIAddS; 3111 } else { 3112 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, count, MIRBuilder); 3113 NegOneReg = 3114 GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); 3115 Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); 3116 Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); 3117 selectOp = SPIRV::OpSelectVIVCond; 3118 addOp = SPIRV::OpIAddV; 3119 } 3120 3121 // check if the high bits are == -1; true if -1 3122 Register BReg = MRI->createVirtualRegister(GR.getRegClass(BoolType)); 3123 Result &= selectNAryOpWithSrcs(BReg, BoolType, I, {HighReg, NegOneReg}, 3124 SPIRV::OpIEqual); 3125 3126 // Select low bits if true in BReg, otherwise high bits 3127 Register TmpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3128 Result &= selectNAryOpWithSrcs(TmpReg, ResType, I, {BReg, LowReg, HighReg}, 3129 selectOp); 3130 3131 // Add 32 for high bits, 0 for low bits 3132 Register ValReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3133 Result &= 3134 selectNAryOpWithSrcs(ValReg, ResType, I, {BReg, Reg0, Reg32}, selectOp); 3135 3136 return Result && 3137 selectNAryOpWithSrcs(ResVReg, ResType, I, {ValReg, TmpReg}, addOp); 3138 } 3139 3140 bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, 3141 const SPIRVType *ResType, 3142 MachineInstr &I, 3143 bool IsSigned) const { 3144 // FindUMsb and FindSMsb intrinsics only support 32 bit integers 3145 Register OpReg = I.getOperand(2).getReg(); 3146 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 3147 3148 switch (GR.getScalarOrVectorBitWidth(OpType)) { 3149 case 16: 3150 return selectFirstBitHigh16(ResVReg, ResType, I, IsSigned); 3151 case 32: 3152 return selectFirstBitHigh32(ResVReg, ResType, I, OpReg, IsSigned); 3153 case 64: 3154 return selectFirstBitHigh64(ResVReg, ResType, I, IsSigned); 3155 default: 3156 report_fatal_error( 3157 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); 3158 } 3159 } 3160 3161 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 3162 const SPIRVType *ResType, 3163 MachineInstr &I) const { 3164 // there was an allocation size parameter to the allocation instruction 3165 // that is not 1 3166 MachineBasicBlock &BB = *I.getParent(); 3167 return BuildMI(BB, I, I.getDebugLoc(), 3168 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 3169 .addDef(ResVReg) 3170 .addUse(GR.getSPIRVTypeID(ResType)) 3171 .addUse(I.getOperand(2).getReg()) 3172 .constrainAllUses(TII, TRI, RBI); 3173 } 3174 3175 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 3176 const SPIRVType *ResType, 3177 MachineInstr &I) const { 3178 // Change order of instructions if needed: all OpVariable instructions in a 3179 // function must be the first instructions in the first block 3180 MachineFunction *MF = I.getParent()->getParent(); 3181 MachineBasicBlock *MBB = &MF->front(); 3182 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 3183 bool IsHeader = false; 3184 unsigned Opcode; 3185 for (; It != E && It != I; ++It) { 3186 Opcode = It->getOpcode(); 3187 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 3188 IsHeader = true; 3189 } else if (IsHeader && 3190 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 3191 ++It; 3192 break; 3193 } 3194 } 3195 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 3196 .addDef(ResVReg) 3197 .addUse(GR.getSPIRVTypeID(ResType)) 3198 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 3199 .constrainAllUses(TII, TRI, RBI); 3200 } 3201 3202 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 3203 // InstructionSelector walks backwards through the instructions. We can use 3204 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 3205 // first, so can generate an OpBranchConditional here. If there is no 3206 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 3207 const MachineInstr *PrevI = I.getPrevNode(); 3208 MachineBasicBlock &MBB = *I.getParent(); 3209 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 3210 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3211 .addUse(PrevI->getOperand(0).getReg()) 3212 .addMBB(PrevI->getOperand(1).getMBB()) 3213 .addMBB(I.getOperand(0).getMBB()) 3214 .constrainAllUses(TII, TRI, RBI); 3215 } 3216 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 3217 .addMBB(I.getOperand(0).getMBB()) 3218 .constrainAllUses(TII, TRI, RBI); 3219 } 3220 3221 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 3222 // InstructionSelector walks backwards through the instructions. For an 3223 // explicit conditional branch with no fallthrough, we use both a G_BR and a 3224 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 3225 // generate the OpBranchConditional in selectBranch above. 3226 // 3227 // If an OpBranchConditional has been generated, we simply return, as the work 3228 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 3229 // implicit fallthrough to the next basic block, so we need to create an 3230 // OpBranchConditional with an explicit "false" argument pointing to the next 3231 // basic block that LLVM would fall through to. 3232 const MachineInstr *NextI = I.getNextNode(); 3233 // Check if this has already been successfully selected. 3234 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 3235 return true; 3236 // Must be relying on implicit block fallthrough, so generate an 3237 // OpBranchConditional with the "next" basic block as the "false" target. 3238 MachineBasicBlock &MBB = *I.getParent(); 3239 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 3240 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 3241 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 3242 .addUse(I.getOperand(0).getReg()) 3243 .addMBB(I.getOperand(1).getMBB()) 3244 .addMBB(NextMBB) 3245 .constrainAllUses(TII, TRI, RBI); 3246 } 3247 3248 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 3249 const SPIRVType *ResType, 3250 MachineInstr &I) const { 3251 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 3252 .addDef(ResVReg) 3253 .addUse(GR.getSPIRVTypeID(ResType)); 3254 const unsigned NumOps = I.getNumOperands(); 3255 for (unsigned i = 1; i < NumOps; i += 2) { 3256 MIB.addUse(I.getOperand(i + 0).getReg()); 3257 MIB.addMBB(I.getOperand(i + 1).getMBB()); 3258 } 3259 return MIB.constrainAllUses(TII, TRI, RBI); 3260 } 3261 3262 bool SPIRVInstructionSelector::selectGlobalValue( 3263 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 3264 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 3265 MachineIRBuilder MIRBuilder(I); 3266 const GlobalValue *GV = I.getOperand(1).getGlobal(); 3267 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 3268 SPIRVType *PointerBaseType; 3269 if (GVType->isArrayTy()) { 3270 SPIRVType *ArrayElementType = 3271 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 3272 SPIRV::AccessQualifier::ReadWrite, false); 3273 PointerBaseType = GR.getOrCreateSPIRVArrayType( 3274 ArrayElementType, GVType->getArrayNumElements(), I, TII); 3275 } else { 3276 PointerBaseType = GR.getOrCreateSPIRVType( 3277 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 3278 } 3279 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 3280 PointerBaseType, I, TII, 3281 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 3282 3283 std::string GlobalIdent; 3284 if (!GV->hasName()) { 3285 unsigned &ID = UnnamedGlobalIDs[GV]; 3286 if (ID == 0) 3287 ID = UnnamedGlobalIDs.size(); 3288 GlobalIdent = "__unnamed_" + Twine(ID).str(); 3289 } else { 3290 GlobalIdent = GV->getGlobalIdentifier(); 3291 } 3292 3293 // Behaviour of functions as operands depends on availability of the 3294 // corresponding extension (SPV_INTEL_function_pointers): 3295 // - If there is an extension to operate with functions as operands: 3296 // We create a proper constant operand and evaluate a correct type for a 3297 // function pointer. 3298 // - Without the required extension: 3299 // We have functions as operands in tests with blocks of instruction e.g. in 3300 // transcoding/global_block.ll. These operands are not used and should be 3301 // substituted by zero constants. Their type is expected to be always 3302 // OpTypePointer Function %uchar. 3303 if (isa<Function>(GV)) { 3304 const Constant *ConstVal = GV; 3305 MachineBasicBlock &BB = *I.getParent(); 3306 Register NewReg = GR.find(ConstVal, GR.CurMF); 3307 if (!NewReg.isValid()) { 3308 Register NewReg = ResVReg; 3309 GR.add(ConstVal, GR.CurMF, NewReg); 3310 const Function *GVFun = 3311 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 3312 ? dyn_cast<Function>(GV) 3313 : nullptr; 3314 if (GVFun) { 3315 // References to a function via function pointers generate virtual 3316 // registers without a definition. We will resolve it later, during 3317 // module analysis stage. 3318 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3319 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 3320 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 3321 MachineInstrBuilder MB = 3322 BuildMI(BB, I, I.getDebugLoc(), 3323 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 3324 .addDef(NewReg) 3325 .addUse(GR.getSPIRVTypeID(ResType)) 3326 .addUse(FuncVReg); 3327 // mapping the function pointer to the used Function 3328 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 3329 return MB.constrainAllUses(TII, TRI, RBI); 3330 } 3331 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 3332 .addDef(NewReg) 3333 .addUse(GR.getSPIRVTypeID(ResType)) 3334 .constrainAllUses(TII, TRI, RBI); 3335 } 3336 assert(NewReg != ResVReg); 3337 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 3338 .addDef(ResVReg) 3339 .addUse(NewReg) 3340 .constrainAllUses(TII, TRI, RBI); 3341 } 3342 auto GlobalVar = cast<GlobalVariable>(GV); 3343 assert(GlobalVar->getName() != "llvm.global.annotations"); 3344 3345 bool HasInit = GlobalVar->hasInitializer() && 3346 !isa<UndefValue>(GlobalVar->getInitializer()); 3347 // Skip empty declaration for GVs with initilaizers till we get the decl with 3348 // passed initializer. 3349 if (HasInit && !Init) 3350 return true; 3351 3352 unsigned AddrSpace = GV->getAddressSpace(); 3353 SPIRV::StorageClass::StorageClass Storage = 3354 addressSpaceToStorageClass(AddrSpace, STI); 3355 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 3356 Storage != SPIRV::StorageClass::Function; 3357 SPIRV::LinkageType::LinkageType LnkType = 3358 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 3359 ? SPIRV::LinkageType::Import 3360 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 3361 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 3362 ? SPIRV::LinkageType::LinkOnceODR 3363 : SPIRV::LinkageType::Export); 3364 3365 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 3366 Storage, Init, GlobalVar->isConstant(), 3367 HasLnkTy, LnkType, MIRBuilder, true); 3368 return Reg.isValid(); 3369 } 3370 3371 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3372 const SPIRVType *ResType, 3373 MachineInstr &I) const { 3374 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3375 return selectExtInst(ResVReg, ResType, I, CL::log10); 3376 } 3377 3378 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3379 // is implemented as: 3380 // log10(x) = log2(x) * (1 / log2(10)) 3381 // = log2(x) * 0.30103 3382 3383 MachineIRBuilder MIRBuilder(I); 3384 MachineBasicBlock &BB = *I.getParent(); 3385 3386 // Build log2(x). 3387 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3388 bool Result = 3389 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3390 .addDef(VarReg) 3391 .addUse(GR.getSPIRVTypeID(ResType)) 3392 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3393 .addImm(GL::Log2) 3394 .add(I.getOperand(1)) 3395 .constrainAllUses(TII, TRI, RBI); 3396 3397 // Build 0.30103. 3398 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3399 ResType->getOpcode() == SPIRV::OpTypeFloat); 3400 // TODO: Add matrix implementation once supported by the HLSL frontend. 3401 const SPIRVType *SpirvScalarType = 3402 ResType->getOpcode() == SPIRV::OpTypeVector 3403 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3404 : ResType; 3405 Register ScaleReg = 3406 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3407 3408 // Multiply log2(x) by 0.30103 to get log10(x) result. 3409 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3410 ? SPIRV::OpVectorTimesScalar 3411 : SPIRV::OpFMulS; 3412 return Result && BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3413 .addDef(ResVReg) 3414 .addUse(GR.getSPIRVTypeID(ResType)) 3415 .addUse(VarReg) 3416 .addUse(ScaleReg) 3417 .constrainAllUses(TII, TRI, RBI); 3418 } 3419 3420 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3421 const SPIRVType *ResType, 3422 MachineInstr &I) const { 3423 // DX intrinsic: @llvm.dx.thread.id(i32) 3424 // ID Name Description 3425 // 93 ThreadId reads the thread ID 3426 3427 MachineIRBuilder MIRBuilder(I); 3428 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3429 const SPIRVType *Vec3Ty = 3430 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3431 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3432 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3433 3434 // Create new register for GlobalInvocationID builtin variable. 3435 Register NewRegister = 3436 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3437 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3438 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3439 3440 // Build GlobalInvocationID global variable with the necessary decorations. 3441 Register Variable = GR.buildGlobalVariable( 3442 NewRegister, PtrType, 3443 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3444 SPIRV::StorageClass::Input, nullptr, true, true, 3445 SPIRV::LinkageType::Import, MIRBuilder, false); 3446 3447 // Create new register for loading value. 3448 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3449 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3450 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3451 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3452 3453 // Load v3uint value from the global variable. 3454 bool Result = 3455 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3456 .addDef(LoadedRegister) 3457 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3458 .addUse(Variable); 3459 3460 // Get Thread ID index. Expecting operand is a constant immediate value, 3461 // wrapped in a type assignment. 3462 assert(I.getOperand(2).isReg()); 3463 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3464 3465 // Extract the thread ID from the loaded vector value. 3466 MachineBasicBlock &BB = *I.getParent(); 3467 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3468 .addDef(ResVReg) 3469 .addUse(GR.getSPIRVTypeID(ResType)) 3470 .addUse(LoadedRegister) 3471 .addImm(ThreadId); 3472 return Result && MIB.constrainAllUses(TII, TRI, RBI); 3473 } 3474 3475 SPIRVType *SPIRVInstructionSelector::widenTypeToVec4(const SPIRVType *Type, 3476 MachineInstr &I) const { 3477 MachineIRBuilder MIRBuilder(I); 3478 if (Type->getOpcode() != SPIRV::OpTypeVector) 3479 return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder); 3480 3481 uint64_t VectorSize = Type->getOperand(2).getImm(); 3482 if (VectorSize == 4) 3483 return Type; 3484 3485 Register ScalarTypeReg = Type->getOperand(1).getReg(); 3486 const SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(ScalarTypeReg); 3487 return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder); 3488 } 3489 3490 namespace llvm { 3491 InstructionSelector * 3492 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3493 const SPIRVSubtarget &Subtarget, 3494 const RegisterBankInfo &RBI) { 3495 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3496 } 3497 } // namespace llvm 3498