1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/Register.h" 32 #include "llvm/CodeGen/TargetOpcodes.h" 33 #include "llvm/IR/IntrinsicsSPIRV.h" 34 #include "llvm/Support/Debug.h" 35 36 #define DEBUG_TYPE "spirv-isel" 37 38 using namespace llvm; 39 namespace CL = SPIRV::OpenCLExtInst; 40 namespace GL = SPIRV::GLSLExtInst; 41 42 using ExtInstList = 43 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 44 45 namespace { 46 47 #define GET_GLOBALISEL_PREDICATE_BITSET 48 #include "SPIRVGenGlobalISel.inc" 49 #undef GET_GLOBALISEL_PREDICATE_BITSET 50 51 class SPIRVInstructionSelector : public InstructionSelector { 52 const SPIRVSubtarget &STI; 53 const SPIRVInstrInfo &TII; 54 const SPIRVRegisterInfo &TRI; 55 const RegisterBankInfo &RBI; 56 SPIRVGlobalRegistry &GR; 57 MachineRegisterInfo *MRI; 58 MachineFunction *HasVRegsReset = nullptr; 59 60 /// We need to keep track of the number we give to anonymous global values to 61 /// generate the same name every time when this is needed. 62 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 63 64 public: 65 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 66 const SPIRVSubtarget &ST, 67 const RegisterBankInfo &RBI); 68 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 69 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 70 BlockFrequencyInfo *BFI) override; 71 // Common selection code. Instruction-specific selection occurs in spvSelect. 72 bool select(MachineInstr &I) override; 73 static const char *getName() { return DEBUG_TYPE; } 74 75 #define GET_GLOBALISEL_PREDICATES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_PREDICATES_DECL 78 79 #define GET_GLOBALISEL_TEMPORARIES_DECL 80 #include "SPIRVGenGlobalISel.inc" 81 #undef GET_GLOBALISEL_TEMPORARIES_DECL 82 83 private: 84 void resetVRegsType(MachineFunction &MF); 85 86 // tblgen-erated 'select' implementation, used as the initial selector for 87 // the patterns that don't require complex C++. 88 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 89 90 // All instruction-specific selection that didn't happen in "select()". 91 // Is basically a large Switch/Case delegating to all other select method. 92 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I) const; 94 95 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 96 const MachineInstr *Init = nullptr) const; 97 98 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I, Register SrcReg, 100 unsigned Opcode) const; 101 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 102 unsigned Opcode) const; 103 104 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 105 MachineInstr &I) const; 106 107 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 108 MachineInstr &I) const; 109 bool selectStore(MachineInstr &I) const; 110 111 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I) const; 113 bool selectStackRestore(MachineInstr &I) const; 114 115 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 116 117 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 118 MachineInstr &I, unsigned NewOpcode, 119 unsigned NegateOpcode = 0) const; 120 121 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 bool selectFence(MachineInstr &I) const; 125 126 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 129 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 130 MachineInstr &I, unsigned OpType) const; 131 132 bool selectAll(Register ResVReg, const SPIRVType *ResType, 133 MachineInstr &I) const; 134 135 bool selectAny(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I) const; 137 138 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 139 MachineInstr &I) const; 140 141 bool selectBuildVector(Register ResVReg, const SPIRVType *ResType, 142 MachineInstr &I) const; 143 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 144 MachineInstr &I) const; 145 146 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 147 unsigned comparisonOpcode, MachineInstr &I) const; 148 bool selectCross(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectSign(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 158 bool selectFloatDot(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 161 bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType, 162 MachineInstr &I, unsigned Opcode) const; 163 164 bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 template <bool Signed> 168 bool selectDot4AddPacked(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 template <bool Signed> 171 bool selectDot4AddPackedExpansion(Register ResVReg, const SPIRVType *ResType, 172 MachineInstr &I) const; 173 174 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 175 int OpIdx) const; 176 void renderFImm64(MachineInstrBuilder &MIB, const MachineInstr &I, 177 int OpIdx) const; 178 179 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 180 MachineInstr &I) const; 181 182 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 183 bool IsSigned) const; 184 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 185 bool IsSigned, unsigned Opcode) const; 186 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 187 bool IsSigned) const; 188 189 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 190 MachineInstr &I) const; 191 192 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 193 const SPIRVType *intTy, const SPIRVType *boolTy) const; 194 195 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 196 MachineInstr &I) const; 197 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 198 MachineInstr &I) const; 199 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 200 MachineInstr &I) const; 201 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 202 MachineInstr &I) const; 203 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 204 MachineInstr &I) const; 205 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I) const; 207 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 208 MachineInstr &I) const; 209 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 210 MachineInstr &I) const; 211 212 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I) const; 214 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 215 MachineInstr &I) const; 216 217 bool selectBranch(MachineInstr &I) const; 218 bool selectBranchCond(MachineInstr &I) const; 219 220 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 223 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 224 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 225 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 226 MachineInstr &I, CL::OpenCLExtInst CLInst, 227 GL::GLSLExtInst GLInst) const; 228 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 229 MachineInstr &I, const ExtInstList &ExtInsts) const; 230 231 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 232 MachineInstr &I) const; 233 234 bool selectSaturate(Register ResVReg, const SPIRVType *ResType, 235 MachineInstr &I) const; 236 237 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 240 bool selectWaveReadLaneAt(Register ResVReg, const SPIRVType *ResType, 241 MachineInstr &I) const; 242 243 bool selectUnmergeValues(MachineInstr &I) const; 244 245 void selectHandleFromBinding(Register &ResVReg, const SPIRVType *ResType, 246 MachineInstr &I) const; 247 248 // Utilities 249 Register buildI32Constant(uint32_t Val, MachineInstr &I, 250 const SPIRVType *ResType = nullptr) const; 251 252 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 253 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 254 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 255 MachineInstr &I) const; 256 Register buildOnesValF(const SPIRVType *ResType, MachineInstr &I) const; 257 258 bool wrapIntoSpecConstantOp(MachineInstr &I, 259 SmallVector<Register> &CompositeArgs) const; 260 261 Register getUcharPtrTypeReg(MachineInstr &I, 262 SPIRV::StorageClass::StorageClass SC) const; 263 MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, 264 Register Src, Register DestType, 265 uint32_t Opcode) const; 266 MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 267 SPIRVType *SrcPtrTy) const; 268 Register buildPointerToResource(const SPIRVType *ResType, uint32_t Set, 269 uint32_t Binding, uint32_t ArraySize, 270 Register IndexReg, bool IsNonUniform, 271 MachineIRBuilder MIRBuilder) const; 272 }; 273 274 } // end anonymous namespace 275 276 #define GET_GLOBALISEL_IMPL 277 #include "SPIRVGenGlobalISel.inc" 278 #undef GET_GLOBALISEL_IMPL 279 280 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 281 const SPIRVSubtarget &ST, 282 const RegisterBankInfo &RBI) 283 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 284 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 285 #define GET_GLOBALISEL_PREDICATES_INIT 286 #include "SPIRVGenGlobalISel.inc" 287 #undef GET_GLOBALISEL_PREDICATES_INIT 288 #define GET_GLOBALISEL_TEMPORARIES_INIT 289 #include "SPIRVGenGlobalISel.inc" 290 #undef GET_GLOBALISEL_TEMPORARIES_INIT 291 { 292 } 293 294 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 295 CodeGenCoverage *CoverageInfo, 296 ProfileSummaryInfo *PSI, 297 BlockFrequencyInfo *BFI) { 298 MRI = &MF.getRegInfo(); 299 GR.setCurrentFunc(MF); 300 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 301 } 302 303 // Ensure that register classes correspond to pattern matching rules. 304 void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { 305 if (HasVRegsReset == &MF) 306 return; 307 HasVRegsReset = &MF; 308 309 MachineRegisterInfo &MRI = MF.getRegInfo(); 310 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { 311 Register Reg = Register::index2VirtReg(I); 312 LLT RegType = MRI.getType(Reg); 313 if (RegType.isScalar()) 314 MRI.setType(Reg, LLT::scalar(64)); 315 else if (RegType.isPointer()) 316 MRI.setType(Reg, LLT::pointer(0, 64)); 317 else if (RegType.isVector()) 318 MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); 319 } 320 for (const auto &MBB : MF) { 321 for (const auto &MI : MBB) { 322 if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) 323 continue; 324 Register DstReg = MI.getOperand(0).getReg(); 325 LLT DstType = MRI.getType(DstReg); 326 Register SrcReg = MI.getOperand(1).getReg(); 327 LLT SrcType = MRI.getType(SrcReg); 328 if (DstType != SrcType) 329 MRI.setType(DstReg, MRI.getType(SrcReg)); 330 331 const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); 332 const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); 333 if (DstRC != SrcRC && SrcRC) 334 MRI.setRegClass(DstReg, SrcRC); 335 } 336 } 337 } 338 339 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 340 341 // Defined in SPIRVLegalizerInfo.cpp. 342 extern bool isTypeFoldingSupported(unsigned Opcode); 343 344 bool SPIRVInstructionSelector::select(MachineInstr &I) { 345 resetVRegsType(*I.getParent()->getParent()); 346 347 assert(I.getParent() && "Instruction should be in a basic block!"); 348 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 349 350 Register Opcode = I.getOpcode(); 351 // If it's not a GMIR instruction, we've selected it already. 352 if (!isPreISelGenericOpcode(Opcode)) { 353 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 354 Register DstReg = I.getOperand(0).getReg(); 355 Register SrcReg = I.getOperand(1).getReg(); 356 auto *Def = MRI->getVRegDef(SrcReg); 357 if (isTypeFoldingSupported(Def->getOpcode())) { 358 bool Res = selectImpl(I, *CoverageInfo); 359 LLVM_DEBUG({ 360 if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { 361 dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; 362 I.print(dbgs()); 363 } 364 }); 365 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 366 if (Res) 367 return Res; 368 } 369 MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); 370 MRI->replaceRegWith(SrcReg, DstReg); 371 I.removeFromParent(); 372 return true; 373 } else if (I.getNumDefs() == 1) { 374 // Make all vregs 64 bits (for SPIR-V IDs). 375 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); 376 } 377 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 378 } 379 380 if (I.getNumOperands() != I.getNumExplicitOperands()) { 381 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 382 return false; 383 } 384 385 // Common code for getting return reg+type, and removing selected instr 386 // from parent occurs here. Instr-specific selection happens in spvSelect(). 387 bool HasDefs = I.getNumDefs() > 0; 388 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 389 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 390 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 391 if (spvSelect(ResVReg, ResType, I)) { 392 if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). 393 for (unsigned i = 0; i < I.getNumDefs(); ++i) 394 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); 395 I.removeFromParent(); 396 return true; 397 } 398 return false; 399 } 400 401 static bool mayApplyGenericSelection(unsigned Opcode) { 402 switch (Opcode) { 403 case TargetOpcode::G_CONSTANT: 404 return false; 405 case TargetOpcode::G_SADDO: 406 case TargetOpcode::G_SSUBO: 407 return true; 408 } 409 return isTypeFoldingSupported(Opcode); 410 } 411 412 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 413 const SPIRVType *ResType, 414 MachineInstr &I) const { 415 const unsigned Opcode = I.getOpcode(); 416 if (mayApplyGenericSelection(Opcode)) 417 return selectImpl(I, *CoverageInfo); 418 switch (Opcode) { 419 case TargetOpcode::G_CONSTANT: 420 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 421 I); 422 case TargetOpcode::G_GLOBAL_VALUE: 423 return selectGlobalValue(ResVReg, I); 424 case TargetOpcode::G_IMPLICIT_DEF: 425 return selectOpUndef(ResVReg, ResType, I); 426 case TargetOpcode::G_FREEZE: 427 return selectFreeze(ResVReg, ResType, I); 428 429 case TargetOpcode::G_INTRINSIC: 430 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 431 case TargetOpcode::G_INTRINSIC_CONVERGENT: 432 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 433 return selectIntrinsic(ResVReg, ResType, I); 434 case TargetOpcode::G_BITREVERSE: 435 return selectBitreverse(ResVReg, ResType, I); 436 437 case TargetOpcode::G_BUILD_VECTOR: 438 return selectBuildVector(ResVReg, ResType, I); 439 case TargetOpcode::G_SPLAT_VECTOR: 440 return selectSplatVector(ResVReg, ResType, I); 441 442 case TargetOpcode::G_SHUFFLE_VECTOR: { 443 MachineBasicBlock &BB = *I.getParent(); 444 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 445 .addDef(ResVReg) 446 .addUse(GR.getSPIRVTypeID(ResType)) 447 .addUse(I.getOperand(1).getReg()) 448 .addUse(I.getOperand(2).getReg()); 449 for (auto V : I.getOperand(3).getShuffleMask()) 450 MIB.addImm(V); 451 return MIB.constrainAllUses(TII, TRI, RBI); 452 } 453 case TargetOpcode::G_MEMMOVE: 454 case TargetOpcode::G_MEMCPY: 455 case TargetOpcode::G_MEMSET: 456 return selectMemOperation(ResVReg, I); 457 458 case TargetOpcode::G_ICMP: 459 return selectICmp(ResVReg, ResType, I); 460 case TargetOpcode::G_FCMP: 461 return selectFCmp(ResVReg, ResType, I); 462 463 case TargetOpcode::G_FRAME_INDEX: 464 return selectFrameIndex(ResVReg, ResType, I); 465 466 case TargetOpcode::G_LOAD: 467 return selectLoad(ResVReg, ResType, I); 468 case TargetOpcode::G_STORE: 469 return selectStore(I); 470 471 case TargetOpcode::G_BR: 472 return selectBranch(I); 473 case TargetOpcode::G_BRCOND: 474 return selectBranchCond(I); 475 476 case TargetOpcode::G_PHI: 477 return selectPhi(ResVReg, ResType, I); 478 479 case TargetOpcode::G_FPTOSI: 480 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 481 case TargetOpcode::G_FPTOUI: 482 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 483 484 case TargetOpcode::G_SITOFP: 485 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 486 case TargetOpcode::G_UITOFP: 487 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 488 489 case TargetOpcode::G_CTPOP: 490 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 491 case TargetOpcode::G_SMIN: 492 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 493 case TargetOpcode::G_UMIN: 494 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 495 496 case TargetOpcode::G_SMAX: 497 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 498 case TargetOpcode::G_UMAX: 499 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 500 501 case TargetOpcode::G_FMA: 502 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 503 504 case TargetOpcode::G_FPOW: 505 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 506 case TargetOpcode::G_FPOWI: 507 return selectExtInst(ResVReg, ResType, I, CL::pown); 508 509 case TargetOpcode::G_FEXP: 510 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 511 case TargetOpcode::G_FEXP2: 512 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 513 514 case TargetOpcode::G_FLOG: 515 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 516 case TargetOpcode::G_FLOG2: 517 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 518 case TargetOpcode::G_FLOG10: 519 return selectLog10(ResVReg, ResType, I); 520 521 case TargetOpcode::G_FABS: 522 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 523 case TargetOpcode::G_ABS: 524 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 525 526 case TargetOpcode::G_FMINNUM: 527 case TargetOpcode::G_FMINIMUM: 528 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 529 case TargetOpcode::G_FMAXNUM: 530 case TargetOpcode::G_FMAXIMUM: 531 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 532 533 case TargetOpcode::G_FCOPYSIGN: 534 return selectExtInst(ResVReg, ResType, I, CL::copysign); 535 536 case TargetOpcode::G_FCEIL: 537 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 538 case TargetOpcode::G_FFLOOR: 539 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 540 541 case TargetOpcode::G_FCOS: 542 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 543 case TargetOpcode::G_FSIN: 544 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 545 case TargetOpcode::G_FTAN: 546 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); 547 case TargetOpcode::G_FACOS: 548 return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); 549 case TargetOpcode::G_FASIN: 550 return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); 551 case TargetOpcode::G_FATAN: 552 return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); 553 case TargetOpcode::G_FATAN2: 554 return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); 555 case TargetOpcode::G_FCOSH: 556 return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); 557 case TargetOpcode::G_FSINH: 558 return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); 559 case TargetOpcode::G_FTANH: 560 return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); 561 562 case TargetOpcode::G_FSQRT: 563 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 564 565 case TargetOpcode::G_CTTZ: 566 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 567 return selectExtInst(ResVReg, ResType, I, CL::ctz); 568 case TargetOpcode::G_CTLZ: 569 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 570 return selectExtInst(ResVReg, ResType, I, CL::clz); 571 572 case TargetOpcode::G_INTRINSIC_ROUND: 573 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 574 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 575 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 576 case TargetOpcode::G_INTRINSIC_TRUNC: 577 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 578 case TargetOpcode::G_FRINT: 579 case TargetOpcode::G_FNEARBYINT: 580 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 581 582 case TargetOpcode::G_SMULH: 583 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 584 case TargetOpcode::G_UMULH: 585 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 586 587 case TargetOpcode::G_SADDSAT: 588 return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); 589 case TargetOpcode::G_UADDSAT: 590 return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); 591 case TargetOpcode::G_SSUBSAT: 592 return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); 593 case TargetOpcode::G_USUBSAT: 594 return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); 595 596 case TargetOpcode::G_UADDO: 597 return selectOverflowArith(ResVReg, ResType, I, 598 ResType->getOpcode() == SPIRV::OpTypeVector 599 ? SPIRV::OpIAddCarryV 600 : SPIRV::OpIAddCarryS); 601 case TargetOpcode::G_USUBO: 602 return selectOverflowArith(ResVReg, ResType, I, 603 ResType->getOpcode() == SPIRV::OpTypeVector 604 ? SPIRV::OpISubBorrowV 605 : SPIRV::OpISubBorrowS); 606 case TargetOpcode::G_UMULO: 607 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); 608 case TargetOpcode::G_SMULO: 609 return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); 610 611 case TargetOpcode::G_SEXT: 612 return selectExt(ResVReg, ResType, I, true); 613 case TargetOpcode::G_ANYEXT: 614 case TargetOpcode::G_ZEXT: 615 return selectExt(ResVReg, ResType, I, false); 616 case TargetOpcode::G_TRUNC: 617 return selectTrunc(ResVReg, ResType, I); 618 case TargetOpcode::G_FPTRUNC: 619 case TargetOpcode::G_FPEXT: 620 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 621 622 case TargetOpcode::G_PTRTOINT: 623 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 624 case TargetOpcode::G_INTTOPTR: 625 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 626 case TargetOpcode::G_BITCAST: 627 return selectBitcast(ResVReg, ResType, I); 628 case TargetOpcode::G_ADDRSPACE_CAST: 629 return selectAddrSpaceCast(ResVReg, ResType, I); 630 case TargetOpcode::G_PTR_ADD: { 631 // Currently, we get G_PTR_ADD only applied to global variables. 632 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 633 Register GV = I.getOperand(1).getReg(); 634 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 635 (void)II; 636 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 637 (*II).getOpcode() == TargetOpcode::COPY || 638 (*II).getOpcode() == SPIRV::OpVariable) && 639 isImm(I.getOperand(2), MRI)); 640 // It may be the initialization of a global variable. 641 bool IsGVInit = false; 642 for (MachineRegisterInfo::use_instr_iterator 643 UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), 644 UseEnd = MRI->use_instr_end(); 645 UseIt != UseEnd; UseIt = std::next(UseIt)) { 646 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 647 (*UseIt).getOpcode() == SPIRV::OpVariable) { 648 IsGVInit = true; 649 break; 650 } 651 } 652 MachineBasicBlock &BB = *I.getParent(); 653 if (!IsGVInit) { 654 SPIRVType *GVType = GR.getSPIRVTypeForVReg(GV); 655 SPIRVType *GVPointeeType = GR.getPointeeType(GVType); 656 SPIRVType *ResPointeeType = GR.getPointeeType(ResType); 657 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { 658 // Build a new virtual register that is associated with the required 659 // data type. 660 Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); 661 MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); 662 // Having a correctly typed base we are ready to build the actually 663 // required GEP. It may not be a constant though, because all Operands 664 // of OpSpecConstantOp is to originate from other const instructions, 665 // and only the AccessChain named opcodes accept a global OpVariable 666 // instruction. We can't use an AccessChain opcode because of the type 667 // mismatch between result and base types. 668 if (!GR.isBitcastCompatible(ResType, GVType)) 669 report_fatal_error( 670 "incompatible result and operand types in a bitcast"); 671 Register ResTypeReg = GR.getSPIRVTypeID(ResType); 672 MachineInstrBuilder MIB = 673 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) 674 .addDef(NewVReg) 675 .addUse(ResTypeReg) 676 .addUse(GV); 677 return MIB.constrainAllUses(TII, TRI, RBI) && 678 BuildMI(BB, I, I.getDebugLoc(), 679 TII.get(STI.isVulkanEnv() 680 ? SPIRV::OpInBoundsAccessChain 681 : SPIRV::OpInBoundsPtrAccessChain)) 682 .addDef(ResVReg) 683 .addUse(ResTypeReg) 684 .addUse(NewVReg) 685 .addUse(I.getOperand(2).getReg()) 686 .constrainAllUses(TII, TRI, RBI); 687 } else { 688 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 689 .addDef(ResVReg) 690 .addUse(GR.getSPIRVTypeID(ResType)) 691 .addImm( 692 static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) 693 .addUse(GV) 694 .addUse(I.getOperand(2).getReg()) 695 .constrainAllUses(TII, TRI, RBI); 696 } 697 } 698 // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to 699 // initialize a global variable with a constant expression (e.g., the test 700 // case opencl/basic/progvar_prog_scope_init.ll), or for another use case 701 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 702 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 703 .addDef(ResVReg) 704 .addUse(GR.getSPIRVTypeID(ResType)) 705 .addImm(static_cast<uint32_t>( 706 SPIRV::Opcode::InBoundsPtrAccessChain)) 707 .addUse(GV) 708 .addUse(Idx) 709 .addUse(I.getOperand(2).getReg()); 710 return MIB.constrainAllUses(TII, TRI, RBI); 711 } 712 713 case TargetOpcode::G_ATOMICRMW_OR: 714 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 715 case TargetOpcode::G_ATOMICRMW_ADD: 716 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 717 case TargetOpcode::G_ATOMICRMW_AND: 718 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 719 case TargetOpcode::G_ATOMICRMW_MAX: 720 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 721 case TargetOpcode::G_ATOMICRMW_MIN: 722 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 723 case TargetOpcode::G_ATOMICRMW_SUB: 724 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 725 case TargetOpcode::G_ATOMICRMW_XOR: 726 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 727 case TargetOpcode::G_ATOMICRMW_UMAX: 728 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 729 case TargetOpcode::G_ATOMICRMW_UMIN: 730 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 731 case TargetOpcode::G_ATOMICRMW_XCHG: 732 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 733 case TargetOpcode::G_ATOMIC_CMPXCHG: 734 return selectAtomicCmpXchg(ResVReg, ResType, I); 735 736 case TargetOpcode::G_ATOMICRMW_FADD: 737 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 738 case TargetOpcode::G_ATOMICRMW_FSUB: 739 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 740 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 741 SPIRV::OpFNegate); 742 case TargetOpcode::G_ATOMICRMW_FMIN: 743 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 744 case TargetOpcode::G_ATOMICRMW_FMAX: 745 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 746 747 case TargetOpcode::G_FENCE: 748 return selectFence(I); 749 750 case TargetOpcode::G_STACKSAVE: 751 return selectStackSave(ResVReg, ResType, I); 752 case TargetOpcode::G_STACKRESTORE: 753 return selectStackRestore(I); 754 755 case TargetOpcode::G_UNMERGE_VALUES: 756 return selectUnmergeValues(I); 757 758 // Discard gen opcodes for intrinsics which we do not expect to actually 759 // represent code after lowering or intrinsics which are not implemented but 760 // should not crash when found in a customer's LLVM IR input. 761 case TargetOpcode::G_TRAP: 762 case TargetOpcode::G_DEBUGTRAP: 763 case TargetOpcode::G_UBSANTRAP: 764 case TargetOpcode::DBG_LABEL: 765 return true; 766 767 default: 768 return false; 769 } 770 } 771 772 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 773 const SPIRVType *ResType, 774 MachineInstr &I, 775 CL::OpenCLExtInst CLInst) const { 776 return selectExtInst(ResVReg, ResType, I, 777 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 778 } 779 780 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 781 const SPIRVType *ResType, 782 MachineInstr &I, 783 CL::OpenCLExtInst CLInst, 784 GL::GLSLExtInst GLInst) const { 785 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 786 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 787 return selectExtInst(ResVReg, ResType, I, ExtInsts); 788 } 789 790 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 791 const SPIRVType *ResType, 792 MachineInstr &I, 793 const ExtInstList &Insts) const { 794 795 for (const auto &Ex : Insts) { 796 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 797 uint32_t Opcode = Ex.second; 798 if (STI.canUseExtInstSet(Set)) { 799 MachineBasicBlock &BB = *I.getParent(); 800 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 801 .addDef(ResVReg) 802 .addUse(GR.getSPIRVTypeID(ResType)) 803 .addImm(static_cast<uint32_t>(Set)) 804 .addImm(Opcode); 805 const unsigned NumOps = I.getNumOperands(); 806 unsigned Index = 1; 807 if (Index < NumOps && 808 I.getOperand(Index).getType() == 809 MachineOperand::MachineOperandType::MO_IntrinsicID) 810 Index = 2; 811 for (; Index < NumOps; ++Index) 812 MIB.add(I.getOperand(Index)); 813 return MIB.constrainAllUses(TII, TRI, RBI); 814 } 815 } 816 return false; 817 } 818 819 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 820 const SPIRVType *ResType, 821 MachineInstr &I, 822 Register SrcReg, 823 unsigned Opcode) const { 824 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 825 .addDef(ResVReg) 826 .addUse(GR.getSPIRVTypeID(ResType)) 827 .addUse(SrcReg) 828 .constrainAllUses(TII, TRI, RBI); 829 } 830 831 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 832 const SPIRVType *ResType, 833 MachineInstr &I, 834 unsigned Opcode) const { 835 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { 836 Register SrcReg = I.getOperand(1).getReg(); 837 bool IsGV = false; 838 for (MachineRegisterInfo::def_instr_iterator DefIt = 839 MRI->def_instr_begin(SrcReg); 840 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { 841 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { 842 IsGV = true; 843 break; 844 } 845 } 846 if (IsGV) { 847 uint32_t SpecOpcode = 0; 848 switch (Opcode) { 849 case SPIRV::OpConvertPtrToU: 850 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); 851 break; 852 case SPIRV::OpConvertUToPtr: 853 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); 854 break; 855 } 856 if (SpecOpcode) 857 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 858 TII.get(SPIRV::OpSpecConstantOp)) 859 .addDef(ResVReg) 860 .addUse(GR.getSPIRVTypeID(ResType)) 861 .addImm(SpecOpcode) 862 .addUse(SrcReg) 863 .constrainAllUses(TII, TRI, RBI); 864 } 865 } 866 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 867 Opcode); 868 } 869 870 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 871 const SPIRVType *ResType, 872 MachineInstr &I) const { 873 Register OpReg = I.getOperand(1).getReg(); 874 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 875 if (!GR.isBitcastCompatible(ResType, OpType)) 876 report_fatal_error("incompatible result and operand types in a bitcast"); 877 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 878 } 879 880 static void addMemoryOperands(MachineMemOperand *MemOp, 881 MachineInstrBuilder &MIB) { 882 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 883 if (MemOp->isVolatile()) 884 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 885 if (MemOp->isNonTemporal()) 886 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 887 if (MemOp->getAlign().value()) 888 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 889 890 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 891 MIB.addImm(SpvMemOp); 892 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 893 MIB.addImm(MemOp->getAlign().value()); 894 } 895 } 896 897 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 898 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 899 if (Flags & MachineMemOperand::Flags::MOVolatile) 900 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 901 if (Flags & MachineMemOperand::Flags::MONonTemporal) 902 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 903 904 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 905 MIB.addImm(SpvMemOp); 906 } 907 908 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 909 const SPIRVType *ResType, 910 MachineInstr &I) const { 911 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 912 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 913 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 914 .addDef(ResVReg) 915 .addUse(GR.getSPIRVTypeID(ResType)) 916 .addUse(Ptr); 917 if (!I.getNumMemOperands()) { 918 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 919 I.getOpcode() == 920 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 921 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 922 } else { 923 addMemoryOperands(*I.memoperands_begin(), MIB); 924 } 925 return MIB.constrainAllUses(TII, TRI, RBI); 926 } 927 928 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 929 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 930 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 931 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 932 MachineBasicBlock &BB = *I.getParent(); 933 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 934 .addUse(Ptr) 935 .addUse(StoreVal); 936 if (!I.getNumMemOperands()) { 937 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 938 I.getOpcode() == 939 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 940 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 941 } else { 942 addMemoryOperands(*I.memoperands_begin(), MIB); 943 } 944 return MIB.constrainAllUses(TII, TRI, RBI); 945 } 946 947 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 948 const SPIRVType *ResType, 949 MachineInstr &I) const { 950 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 951 report_fatal_error( 952 "llvm.stacksave intrinsic: this instruction requires the following " 953 "SPIR-V extension: SPV_INTEL_variable_length_array", 954 false); 955 MachineBasicBlock &BB = *I.getParent(); 956 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 957 .addDef(ResVReg) 958 .addUse(GR.getSPIRVTypeID(ResType)) 959 .constrainAllUses(TII, TRI, RBI); 960 } 961 962 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 963 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 964 report_fatal_error( 965 "llvm.stackrestore intrinsic: this instruction requires the following " 966 "SPIR-V extension: SPV_INTEL_variable_length_array", 967 false); 968 if (!I.getOperand(0).isReg()) 969 return false; 970 MachineBasicBlock &BB = *I.getParent(); 971 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 972 .addUse(I.getOperand(0).getReg()) 973 .constrainAllUses(TII, TRI, RBI); 974 } 975 976 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 977 MachineInstr &I) const { 978 MachineBasicBlock &BB = *I.getParent(); 979 Register SrcReg = I.getOperand(1).getReg(); 980 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 981 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 982 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 983 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 984 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 985 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 986 Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); 987 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 988 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 989 // TODO: check if we have such GV, add init, use buildGlobalVariable. 990 Function &CurFunction = GR.CurMF->getFunction(); 991 Type *LLVMArrTy = 992 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 993 // Module takes ownership of the global var. 994 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 995 true, GlobalValue::InternalLinkage, 996 Constant::getNullValue(LLVMArrTy)); 997 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 998 GR.add(GV, GR.CurMF, VarReg); 999 1000 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1001 .addDef(VarReg) 1002 .addUse(GR.getSPIRVTypeID(VarTy)) 1003 .addImm(SPIRV::StorageClass::UniformConstant) 1004 .addUse(Const) 1005 .constrainAllUses(TII, TRI, RBI); 1006 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 1007 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 1008 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 1009 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1010 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 1011 } 1012 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 1013 .addUse(I.getOperand(0).getReg()) 1014 .addUse(SrcReg) 1015 .addUse(I.getOperand(2).getReg()); 1016 if (I.getNumMemOperands()) 1017 addMemoryOperands(*I.memoperands_begin(), MIB); 1018 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 1019 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 1020 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 1021 .addUse(MIB->getOperand(0).getReg()); 1022 return Result; 1023 } 1024 1025 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 1026 const SPIRVType *ResType, 1027 MachineInstr &I, 1028 unsigned NewOpcode, 1029 unsigned NegateOpcode) const { 1030 assert(I.hasOneMemOperand()); 1031 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1032 uint32_t Scope = static_cast<uint32_t>(getMemScope( 1033 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1034 Register ScopeReg = buildI32Constant(Scope, I); 1035 1036 Register Ptr = I.getOperand(1).getReg(); 1037 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 1038 // auto ScSem = 1039 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 1040 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1041 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1042 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 1043 1044 bool Result = false; 1045 Register ValueReg = I.getOperand(2).getReg(); 1046 if (NegateOpcode != 0) { 1047 // Translation with negative value operand is requested 1048 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1049 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 1050 ValueReg = TmpReg; 1051 } 1052 1053 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 1054 .addDef(ResVReg) 1055 .addUse(GR.getSPIRVTypeID(ResType)) 1056 .addUse(Ptr) 1057 .addUse(ScopeReg) 1058 .addUse(MemSemReg) 1059 .addUse(ValueReg) 1060 .constrainAllUses(TII, TRI, RBI); 1061 return Result; 1062 } 1063 1064 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 1065 unsigned ArgI = I.getNumOperands() - 1; 1066 Register SrcReg = 1067 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 1068 SPIRVType *DefType = 1069 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 1070 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 1071 report_fatal_error( 1072 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 1073 1074 SPIRVType *ScalarType = 1075 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 1076 MachineBasicBlock &BB = *I.getParent(); 1077 bool Res = false; 1078 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1079 Register ResVReg = I.getOperand(i).getReg(); 1080 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 1081 if (!ResType) { 1082 // There was no "assign type" actions, let's fix this now 1083 ResType = ScalarType; 1084 MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); 1085 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 1086 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 1087 } 1088 auto MIB = 1089 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1090 .addDef(ResVReg) 1091 .addUse(GR.getSPIRVTypeID(ResType)) 1092 .addUse(SrcReg) 1093 .addImm(static_cast<int64_t>(i)); 1094 Res |= MIB.constrainAllUses(TII, TRI, RBI); 1095 } 1096 return Res; 1097 } 1098 1099 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 1100 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 1101 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 1102 Register MemSemReg = buildI32Constant(MemSem, I); 1103 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 1104 uint32_t Scope = static_cast<uint32_t>( 1105 getMemScope(GR.CurMF->getFunction().getContext(), Ord)); 1106 Register ScopeReg = buildI32Constant(Scope, I); 1107 MachineBasicBlock &BB = *I.getParent(); 1108 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 1109 .addUse(ScopeReg) 1110 .addUse(MemSemReg) 1111 .constrainAllUses(TII, TRI, RBI); 1112 } 1113 1114 bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, 1115 const SPIRVType *ResType, 1116 MachineInstr &I, 1117 unsigned Opcode) const { 1118 Type *ResTy = nullptr; 1119 StringRef ResName; 1120 if (!GR.findValueAttrs(&I, ResTy, ResName)) 1121 report_fatal_error( 1122 "Not enough info to select the arithmetic with overflow instruction"); 1123 if (!ResTy || !ResTy->isStructTy()) 1124 report_fatal_error("Expect struct type result for the arithmetic " 1125 "with overflow instruction"); 1126 // "Result Type must be from OpTypeStruct. The struct must have two members, 1127 // and the two members must be the same type." 1128 Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); 1129 ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy}); 1130 // Build SPIR-V types and constant(s) if needed. 1131 MachineIRBuilder MIRBuilder(I); 1132 SPIRVType *StructType = GR.getOrCreateSPIRVType( 1133 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1134 assert(I.getNumDefs() > 1 && "Not enought operands"); 1135 SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII); 1136 unsigned N = GR.getScalarOrVectorComponentCount(ResType); 1137 if (N > 1) 1138 BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); 1139 Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); 1140 Register ZeroReg = buildZerosVal(ResType, I); 1141 // A new virtual register to store the result struct. 1142 Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1143 MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); 1144 // Build the result name if needed. 1145 if (ResName.size() > 0) 1146 buildOpName(StructVReg, ResName, MIRBuilder); 1147 // Build the arithmetic with overflow instruction. 1148 MachineBasicBlock &BB = *I.getParent(); 1149 auto MIB = 1150 BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) 1151 .addDef(StructVReg) 1152 .addUse(GR.getSPIRVTypeID(StructType)); 1153 for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) 1154 MIB.addUse(I.getOperand(i).getReg()); 1155 bool Status = MIB.constrainAllUses(TII, TRI, RBI); 1156 // Build instructions to extract fields of the instruction's result. 1157 // A new virtual register to store the higher part of the result struct. 1158 Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 1159 MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); 1160 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 1161 auto MIB = 1162 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1163 .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) 1164 .addUse(GR.getSPIRVTypeID(ResType)) 1165 .addUse(StructVReg) 1166 .addImm(i); 1167 Status &= MIB.constrainAllUses(TII, TRI, RBI); 1168 } 1169 // Build boolean value from the higher part. 1170 Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1171 .addDef(I.getOperand(1).getReg()) 1172 .addUse(BoolTypeReg) 1173 .addUse(HigherVReg) 1174 .addUse(ZeroReg) 1175 .constrainAllUses(TII, TRI, RBI); 1176 return Status; 1177 } 1178 1179 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 1180 const SPIRVType *ResType, 1181 MachineInstr &I) const { 1182 Register ScopeReg; 1183 Register MemSemEqReg; 1184 Register MemSemNeqReg; 1185 Register Ptr = I.getOperand(2).getReg(); 1186 if (!isa<GIntrinsic>(I)) { 1187 assert(I.hasOneMemOperand()); 1188 const MachineMemOperand *MemOp = *I.memoperands_begin(); 1189 unsigned Scope = static_cast<uint32_t>(getMemScope( 1190 GR.CurMF->getFunction().getContext(), MemOp->getSyncScopeID())); 1191 ScopeReg = buildI32Constant(Scope, I); 1192 1193 unsigned ScSem = static_cast<uint32_t>( 1194 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 1195 AtomicOrdering AO = MemOp->getSuccessOrdering(); 1196 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 1197 MemSemEqReg = buildI32Constant(MemSemEq, I); 1198 AtomicOrdering FO = MemOp->getFailureOrdering(); 1199 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 1200 MemSemNeqReg = 1201 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 1202 } else { 1203 ScopeReg = I.getOperand(5).getReg(); 1204 MemSemEqReg = I.getOperand(6).getReg(); 1205 MemSemNeqReg = I.getOperand(7).getReg(); 1206 } 1207 1208 Register Cmp = I.getOperand(3).getReg(); 1209 Register Val = I.getOperand(4).getReg(); 1210 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 1211 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1212 const DebugLoc &DL = I.getDebugLoc(); 1213 bool Result = 1214 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 1215 .addDef(ACmpRes) 1216 .addUse(GR.getSPIRVTypeID(SpvValTy)) 1217 .addUse(Ptr) 1218 .addUse(ScopeReg) 1219 .addUse(MemSemEqReg) 1220 .addUse(MemSemNeqReg) 1221 .addUse(Val) 1222 .addUse(Cmp) 1223 .constrainAllUses(TII, TRI, RBI); 1224 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1225 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 1226 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 1227 .addDef(CmpSuccReg) 1228 .addUse(GR.getSPIRVTypeID(BoolTy)) 1229 .addUse(ACmpRes) 1230 .addUse(Cmp) 1231 .constrainAllUses(TII, TRI, RBI); 1232 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1233 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1234 .addDef(TmpReg) 1235 .addUse(GR.getSPIRVTypeID(ResType)) 1236 .addUse(ACmpRes) 1237 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 1238 .addImm(0) 1239 .constrainAllUses(TII, TRI, RBI); 1240 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 1241 .addDef(ResVReg) 1242 .addUse(GR.getSPIRVTypeID(ResType)) 1243 .addUse(CmpSuccReg) 1244 .addUse(TmpReg) 1245 .addImm(1) 1246 .constrainAllUses(TII, TRI, RBI); 1247 return Result; 1248 } 1249 1250 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 1251 switch (SC) { 1252 case SPIRV::StorageClass::Workgroup: 1253 case SPIRV::StorageClass::CrossWorkgroup: 1254 case SPIRV::StorageClass::Function: 1255 return true; 1256 default: 1257 return false; 1258 } 1259 } 1260 1261 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 1262 switch (SC) { 1263 case SPIRV::StorageClass::DeviceOnlyINTEL: 1264 case SPIRV::StorageClass::HostOnlyINTEL: 1265 return true; 1266 default: 1267 return false; 1268 } 1269 } 1270 1271 // Returns true ResVReg is referred only from global vars and OpName's. 1272 static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { 1273 bool IsGRef = false; 1274 bool IsAllowedRefs = 1275 std::all_of(MRI->use_instr_begin(ResVReg), MRI->use_instr_end(), 1276 [&IsGRef](auto const &It) { 1277 unsigned Opcode = It.getOpcode(); 1278 if (Opcode == SPIRV::OpConstantComposite || 1279 Opcode == SPIRV::OpVariable || 1280 isSpvIntrinsic(It, Intrinsic::spv_init_global)) 1281 return IsGRef = true; 1282 return Opcode == SPIRV::OpName; 1283 }); 1284 return IsAllowedRefs && IsGRef; 1285 } 1286 1287 Register SPIRVInstructionSelector::getUcharPtrTypeReg( 1288 MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { 1289 return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( 1290 GR.getOrCreateSPIRVIntegerType(8, I, TII), I, TII, SC)); 1291 } 1292 1293 MachineInstrBuilder 1294 SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, 1295 Register Src, Register DestType, 1296 uint32_t Opcode) const { 1297 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1298 TII.get(SPIRV::OpSpecConstantOp)) 1299 .addDef(Dest) 1300 .addUse(DestType) 1301 .addImm(Opcode) 1302 .addUse(Src); 1303 } 1304 1305 MachineInstrBuilder 1306 SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, 1307 SPIRVType *SrcPtrTy) const { 1308 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1309 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1310 Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); 1311 MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( 1312 SPIRV::StorageClass::Generic), 1313 GR.getPointerSize())); 1314 MachineFunction *MF = I.getParent()->getParent(); 1315 GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); 1316 MachineInstrBuilder MIB = buildSpecConstantOp( 1317 I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), 1318 static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); 1319 GR.add(MIB.getInstr(), MF, Tmp); 1320 return MIB; 1321 } 1322 1323 // In SPIR-V address space casting can only happen to and from the Generic 1324 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1325 // pointers to and from Generic pointers. As such, we can convert e.g. from 1326 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1327 // other combinations can only be done by a bitcast, and are probably not safe. 1328 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1329 const SPIRVType *ResType, 1330 MachineInstr &I) const { 1331 MachineBasicBlock &BB = *I.getParent(); 1332 const DebugLoc &DL = I.getDebugLoc(); 1333 1334 Register SrcPtr = I.getOperand(1).getReg(); 1335 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1336 1337 // don't generate a cast for a null that may be represented by OpTypeInt 1338 if (SrcPtrTy->getOpcode() != SPIRV::OpTypePointer || 1339 ResType->getOpcode() != SPIRV::OpTypePointer) 1340 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1341 .addDef(ResVReg) 1342 .addUse(SrcPtr) 1343 .constrainAllUses(TII, TRI, RBI); 1344 1345 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); 1346 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); 1347 1348 if (isASCastInGVar(MRI, ResVReg)) { 1349 // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions 1350 // are expressed by OpSpecConstantOp with an Opcode. 1351 // TODO: maybe insert a check whether the Kernel capability was declared and 1352 // so PtrCastToGeneric/GenericCastToPtr are available. 1353 unsigned SpecOpcode = 1354 DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC) 1355 ? static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric) 1356 : (SrcSC == SPIRV::StorageClass::Generic && 1357 isGenericCastablePtr(DstSC) 1358 ? static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr) 1359 : 0); 1360 // TODO: OpConstantComposite expects i8*, so we are forced to forget a 1361 // correct value of ResType and use general i8* instead. Maybe this should 1362 // be addressed in the emit-intrinsic step to infer a correct 1363 // OpConstantComposite type. 1364 if (SpecOpcode) { 1365 return buildSpecConstantOp(I, ResVReg, SrcPtr, 1366 getUcharPtrTypeReg(I, DstSC), SpecOpcode) 1367 .constrainAllUses(TII, TRI, RBI); 1368 } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1369 MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); 1370 return MIB.constrainAllUses(TII, TRI, RBI) && 1371 buildSpecConstantOp( 1372 I, ResVReg, MIB->getOperand(0).getReg(), 1373 getUcharPtrTypeReg(I, DstSC), 1374 static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) 1375 .constrainAllUses(TII, TRI, RBI); 1376 } 1377 } 1378 1379 // don't generate a cast between identical storage classes 1380 if (SrcSC == DstSC) 1381 return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY)) 1382 .addDef(ResVReg) 1383 .addUse(SrcPtr) 1384 .constrainAllUses(TII, TRI, RBI); 1385 1386 // Casting from an eligible pointer to Generic. 1387 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1388 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1389 // Casting from Generic to an eligible pointer. 1390 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1391 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1392 // Casting between 2 eligible pointers using Generic as an intermediary. 1393 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1394 Register Tmp = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1395 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1396 GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); 1397 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1398 .addDef(Tmp) 1399 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1400 .addUse(SrcPtr) 1401 .constrainAllUses(TII, TRI, RBI); 1402 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1403 .addDef(ResVReg) 1404 .addUse(GR.getSPIRVTypeID(ResType)) 1405 .addUse(Tmp) 1406 .constrainAllUses(TII, TRI, RBI); 1407 } 1408 1409 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1410 // be applied 1411 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1412 return selectUnOp(ResVReg, ResType, I, 1413 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1414 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1415 return selectUnOp(ResVReg, ResType, I, 1416 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1417 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) 1418 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1419 if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) 1420 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1421 1422 // Bitcast for pointers requires that the address spaces must match 1423 return false; 1424 } 1425 1426 static unsigned getFCmpOpcode(unsigned PredNum) { 1427 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1428 switch (Pred) { 1429 case CmpInst::FCMP_OEQ: 1430 return SPIRV::OpFOrdEqual; 1431 case CmpInst::FCMP_OGE: 1432 return SPIRV::OpFOrdGreaterThanEqual; 1433 case CmpInst::FCMP_OGT: 1434 return SPIRV::OpFOrdGreaterThan; 1435 case CmpInst::FCMP_OLE: 1436 return SPIRV::OpFOrdLessThanEqual; 1437 case CmpInst::FCMP_OLT: 1438 return SPIRV::OpFOrdLessThan; 1439 case CmpInst::FCMP_ONE: 1440 return SPIRV::OpFOrdNotEqual; 1441 case CmpInst::FCMP_ORD: 1442 return SPIRV::OpOrdered; 1443 case CmpInst::FCMP_UEQ: 1444 return SPIRV::OpFUnordEqual; 1445 case CmpInst::FCMP_UGE: 1446 return SPIRV::OpFUnordGreaterThanEqual; 1447 case CmpInst::FCMP_UGT: 1448 return SPIRV::OpFUnordGreaterThan; 1449 case CmpInst::FCMP_ULE: 1450 return SPIRV::OpFUnordLessThanEqual; 1451 case CmpInst::FCMP_ULT: 1452 return SPIRV::OpFUnordLessThan; 1453 case CmpInst::FCMP_UNE: 1454 return SPIRV::OpFUnordNotEqual; 1455 case CmpInst::FCMP_UNO: 1456 return SPIRV::OpUnordered; 1457 default: 1458 llvm_unreachable("Unknown predicate type for FCmp"); 1459 } 1460 } 1461 1462 static unsigned getICmpOpcode(unsigned PredNum) { 1463 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1464 switch (Pred) { 1465 case CmpInst::ICMP_EQ: 1466 return SPIRV::OpIEqual; 1467 case CmpInst::ICMP_NE: 1468 return SPIRV::OpINotEqual; 1469 case CmpInst::ICMP_SGE: 1470 return SPIRV::OpSGreaterThanEqual; 1471 case CmpInst::ICMP_SGT: 1472 return SPIRV::OpSGreaterThan; 1473 case CmpInst::ICMP_SLE: 1474 return SPIRV::OpSLessThanEqual; 1475 case CmpInst::ICMP_SLT: 1476 return SPIRV::OpSLessThan; 1477 case CmpInst::ICMP_UGE: 1478 return SPIRV::OpUGreaterThanEqual; 1479 case CmpInst::ICMP_UGT: 1480 return SPIRV::OpUGreaterThan; 1481 case CmpInst::ICMP_ULE: 1482 return SPIRV::OpULessThanEqual; 1483 case CmpInst::ICMP_ULT: 1484 return SPIRV::OpULessThan; 1485 default: 1486 llvm_unreachable("Unknown predicate type for ICmp"); 1487 } 1488 } 1489 1490 static unsigned getPtrCmpOpcode(unsigned Pred) { 1491 switch (static_cast<CmpInst::Predicate>(Pred)) { 1492 case CmpInst::ICMP_EQ: 1493 return SPIRV::OpPtrEqual; 1494 case CmpInst::ICMP_NE: 1495 return SPIRV::OpPtrNotEqual; 1496 default: 1497 llvm_unreachable("Unknown predicate type for pointer comparison"); 1498 } 1499 } 1500 1501 // Return the logical operation, or abort if none exists. 1502 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1503 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1504 switch (Pred) { 1505 case CmpInst::ICMP_EQ: 1506 return SPIRV::OpLogicalEqual; 1507 case CmpInst::ICMP_NE: 1508 return SPIRV::OpLogicalNotEqual; 1509 default: 1510 llvm_unreachable("Unknown predicate type for Bool comparison"); 1511 } 1512 } 1513 1514 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1515 if (!LLVMFloatTy) 1516 return APFloat::getZero(APFloat::IEEEsingle()); 1517 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1518 case Type::HalfTyID: 1519 return APFloat::getZero(APFloat::IEEEhalf()); 1520 default: 1521 case Type::FloatTyID: 1522 return APFloat::getZero(APFloat::IEEEsingle()); 1523 case Type::DoubleTyID: 1524 return APFloat::getZero(APFloat::IEEEdouble()); 1525 } 1526 } 1527 1528 static APFloat getOneFP(const Type *LLVMFloatTy) { 1529 if (!LLVMFloatTy) 1530 return APFloat::getOne(APFloat::IEEEsingle()); 1531 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1532 case Type::HalfTyID: 1533 return APFloat::getOne(APFloat::IEEEhalf()); 1534 default: 1535 case Type::FloatTyID: 1536 return APFloat::getOne(APFloat::IEEEsingle()); 1537 case Type::DoubleTyID: 1538 return APFloat::getOne(APFloat::IEEEdouble()); 1539 } 1540 } 1541 1542 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1543 const SPIRVType *ResType, 1544 MachineInstr &I, 1545 unsigned OpAnyOrAll) const { 1546 assert(I.getNumOperands() == 3); 1547 assert(I.getOperand(2).isReg()); 1548 MachineBasicBlock &BB = *I.getParent(); 1549 Register InputRegister = I.getOperand(2).getReg(); 1550 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1551 1552 if (!InputType) 1553 report_fatal_error("Input Type could not be determined."); 1554 1555 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1556 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1557 if (IsBoolTy && !IsVectorTy) { 1558 assert(ResVReg == I.getOperand(0).getReg()); 1559 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1560 TII.get(TargetOpcode::COPY)) 1561 .addDef(ResVReg) 1562 .addUse(InputRegister) 1563 .constrainAllUses(TII, TRI, RBI); 1564 } 1565 1566 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1567 unsigned SpirvNotEqualId = 1568 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1569 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1570 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1571 Register NotEqualReg = ResVReg; 1572 1573 if (IsVectorTy) { 1574 NotEqualReg = IsBoolTy ? InputRegister 1575 : MRI->createVirtualRegister(&SPIRV::iIDRegClass); 1576 const unsigned NumElts = InputType->getOperand(2).getImm(); 1577 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1578 } 1579 1580 if (!IsBoolTy) { 1581 Register ConstZeroReg = 1582 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1583 1584 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1585 .addDef(NotEqualReg) 1586 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1587 .addUse(InputRegister) 1588 .addUse(ConstZeroReg) 1589 .constrainAllUses(TII, TRI, RBI); 1590 } 1591 1592 if (!IsVectorTy) 1593 return true; 1594 1595 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1596 .addDef(ResVReg) 1597 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1598 .addUse(NotEqualReg) 1599 .constrainAllUses(TII, TRI, RBI); 1600 } 1601 1602 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1603 const SPIRVType *ResType, 1604 MachineInstr &I) const { 1605 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1606 } 1607 1608 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1609 const SPIRVType *ResType, 1610 MachineInstr &I) const { 1611 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1612 } 1613 1614 // Select the OpDot instruction for the given float dot 1615 bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, 1616 const SPIRVType *ResType, 1617 MachineInstr &I) const { 1618 assert(I.getNumOperands() == 4); 1619 assert(I.getOperand(2).isReg()); 1620 assert(I.getOperand(3).isReg()); 1621 1622 [[maybe_unused]] SPIRVType *VecType = 1623 GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); 1624 1625 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1626 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1627 "dot product requires a vector of at least 2 components"); 1628 1629 [[maybe_unused]] SPIRVType *EltType = 1630 GR.getSPIRVTypeForVReg(VecType->getOperand(1).getReg()); 1631 1632 assert(EltType->getOpcode() == SPIRV::OpTypeFloat); 1633 1634 MachineBasicBlock &BB = *I.getParent(); 1635 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) 1636 .addDef(ResVReg) 1637 .addUse(GR.getSPIRVTypeID(ResType)) 1638 .addUse(I.getOperand(2).getReg()) 1639 .addUse(I.getOperand(3).getReg()) 1640 .constrainAllUses(TII, TRI, RBI); 1641 } 1642 1643 // Since pre-1.6 SPIRV has no integer dot implementation, 1644 // expand by piecewise multiplying and adding the results 1645 bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, 1646 const SPIRVType *ResType, 1647 MachineInstr &I) const { 1648 assert(I.getNumOperands() == 4); 1649 assert(I.getOperand(2).isReg()); 1650 assert(I.getOperand(3).isReg()); 1651 MachineBasicBlock &BB = *I.getParent(); 1652 1653 // Multiply the vectors, then sum the results 1654 Register Vec0 = I.getOperand(2).getReg(); 1655 Register Vec1 = I.getOperand(3).getReg(); 1656 Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1657 SPIRVType *VecType = GR.getSPIRVTypeForVReg(Vec0); 1658 1659 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) 1660 .addDef(TmpVec) 1661 .addUse(GR.getSPIRVTypeID(VecType)) 1662 .addUse(Vec0) 1663 .addUse(Vec1) 1664 .constrainAllUses(TII, TRI, RBI); 1665 1666 assert(VecType->getOpcode() == SPIRV::OpTypeVector && 1667 GR.getScalarOrVectorComponentCount(VecType) > 1 && 1668 "dot product requires a vector of at least 2 components"); 1669 1670 Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1671 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1672 .addDef(Res) 1673 .addUse(GR.getSPIRVTypeID(ResType)) 1674 .addUse(TmpVec) 1675 .addImm(0) 1676 .constrainAllUses(TII, TRI, RBI); 1677 1678 for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { 1679 Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1680 1681 Result &= 1682 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1683 .addDef(Elt) 1684 .addUse(GR.getSPIRVTypeID(ResType)) 1685 .addUse(TmpVec) 1686 .addImm(i) 1687 .constrainAllUses(TII, TRI, RBI); 1688 1689 Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 1690 ? MRI->createVirtualRegister(GR.getRegClass(ResType)) 1691 : ResVReg; 1692 1693 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1694 .addDef(Sum) 1695 .addUse(GR.getSPIRVTypeID(ResType)) 1696 .addUse(Res) 1697 .addUse(Elt) 1698 .constrainAllUses(TII, TRI, RBI); 1699 Res = Sum; 1700 } 1701 1702 return Result; 1703 } 1704 1705 template <bool Signed> 1706 bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, 1707 const SPIRVType *ResType, 1708 MachineInstr &I) const { 1709 assert(I.getNumOperands() == 5); 1710 assert(I.getOperand(2).isReg()); 1711 assert(I.getOperand(3).isReg()); 1712 assert(I.getOperand(4).isReg()); 1713 MachineBasicBlock &BB = *I.getParent(); 1714 1715 auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; 1716 Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); 1717 bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) 1718 .addDef(Dot) 1719 .addUse(GR.getSPIRVTypeID(ResType)) 1720 .addUse(I.getOperand(2).getReg()) 1721 .addUse(I.getOperand(3).getReg()) 1722 .constrainAllUses(TII, TRI, RBI); 1723 1724 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1725 .addDef(ResVReg) 1726 .addUse(GR.getSPIRVTypeID(ResType)) 1727 .addUse(Dot) 1728 .addUse(I.getOperand(4).getReg()) 1729 .constrainAllUses(TII, TRI, RBI); 1730 1731 return Result; 1732 } 1733 1734 // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, 1735 // extract the elements of the packed inputs, multiply them and add the result 1736 // to the accumulator. 1737 template <bool Signed> 1738 bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( 1739 Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { 1740 assert(I.getNumOperands() == 5); 1741 assert(I.getOperand(2).isReg()); 1742 assert(I.getOperand(3).isReg()); 1743 assert(I.getOperand(4).isReg()); 1744 MachineBasicBlock &BB = *I.getParent(); 1745 1746 bool Result = false; 1747 1748 // Acc = C 1749 Register Acc = I.getOperand(4).getReg(); 1750 SPIRVType *EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1751 auto ExtractOp = 1752 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; 1753 1754 // Extract the i8 element, multiply and add it to the accumulator 1755 for (unsigned i = 0; i < 4; i++) { 1756 // A[i] 1757 Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1758 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1759 .addDef(AElt) 1760 .addUse(GR.getSPIRVTypeID(ResType)) 1761 .addUse(I.getOperand(2).getReg()) 1762 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1763 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1764 .constrainAllUses(TII, TRI, RBI); 1765 1766 // B[i] 1767 Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1768 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1769 .addDef(BElt) 1770 .addUse(GR.getSPIRVTypeID(ResType)) 1771 .addUse(I.getOperand(3).getReg()) 1772 .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII)) 1773 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1774 .constrainAllUses(TII, TRI, RBI); 1775 1776 // A[i] * B[i] 1777 Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1778 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) 1779 .addDef(Mul) 1780 .addUse(GR.getSPIRVTypeID(ResType)) 1781 .addUse(AElt) 1782 .addUse(BElt) 1783 .constrainAllUses(TII, TRI, RBI); 1784 1785 // Discard 24 highest-bits so that stored i32 register is i8 equivalent 1786 Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1787 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) 1788 .addDef(MaskMul) 1789 .addUse(GR.getSPIRVTypeID(ResType)) 1790 .addUse(Mul) 1791 .addUse(GR.getOrCreateConstInt(0, I, EltType, TII)) 1792 .addUse(GR.getOrCreateConstInt(8, I, EltType, TII)) 1793 .constrainAllUses(TII, TRI, RBI); 1794 1795 // Acc = Acc + A[i] * B[i] 1796 Register Sum = 1797 i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; 1798 Result |= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) 1799 .addDef(Sum) 1800 .addUse(GR.getSPIRVTypeID(ResType)) 1801 .addUse(Acc) 1802 .addUse(MaskMul) 1803 .constrainAllUses(TII, TRI, RBI); 1804 1805 Acc = Sum; 1806 } 1807 1808 return Result; 1809 } 1810 1811 /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV 1812 /// does not have a saturate builtin. 1813 bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, 1814 const SPIRVType *ResType, 1815 MachineInstr &I) const { 1816 assert(I.getNumOperands() == 3); 1817 assert(I.getOperand(2).isReg()); 1818 MachineBasicBlock &BB = *I.getParent(); 1819 Register VZero = buildZerosValF(ResType, I); 1820 Register VOne = buildOnesValF(ResType, I); 1821 1822 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1823 .addDef(ResVReg) 1824 .addUse(GR.getSPIRVTypeID(ResType)) 1825 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1826 .addImm(GL::FClamp) 1827 .addUse(I.getOperand(2).getReg()) 1828 .addUse(VZero) 1829 .addUse(VOne) 1830 .constrainAllUses(TII, TRI, RBI); 1831 } 1832 1833 bool SPIRVInstructionSelector::selectSign(Register ResVReg, 1834 const SPIRVType *ResType, 1835 MachineInstr &I) const { 1836 assert(I.getNumOperands() == 3); 1837 assert(I.getOperand(2).isReg()); 1838 MachineBasicBlock &BB = *I.getParent(); 1839 Register InputRegister = I.getOperand(2).getReg(); 1840 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1841 auto &DL = I.getDebugLoc(); 1842 1843 if (!InputType) 1844 report_fatal_error("Input Type could not be determined."); 1845 1846 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1847 1848 unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); 1849 unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); 1850 1851 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; 1852 1853 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; 1854 Register SignReg = NeedsConversion 1855 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) 1856 : ResVReg; 1857 1858 bool Result = 1859 BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) 1860 .addDef(SignReg) 1861 .addUse(GR.getSPIRVTypeID(InputType)) 1862 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1863 .addImm(SignOpcode) 1864 .addUse(InputRegister) 1865 .constrainAllUses(TII, TRI, RBI); 1866 1867 if (NeedsConversion) { 1868 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; 1869 Result |= BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) 1870 .addDef(ResVReg) 1871 .addUse(GR.getSPIRVTypeID(ResType)) 1872 .addUse(SignReg) 1873 .constrainAllUses(TII, TRI, RBI); 1874 } 1875 1876 return Result; 1877 } 1878 1879 bool SPIRVInstructionSelector::selectWaveReadLaneAt(Register ResVReg, 1880 const SPIRVType *ResType, 1881 MachineInstr &I) const { 1882 assert(I.getNumOperands() == 4); 1883 assert(I.getOperand(2).isReg()); 1884 assert(I.getOperand(3).isReg()); 1885 MachineBasicBlock &BB = *I.getParent(); 1886 1887 // IntTy is used to define the execution scope, set to 3 to denote a 1888 // cross-lane interaction equivalent to a SPIR-V subgroup. 1889 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 1890 return BuildMI(BB, I, I.getDebugLoc(), 1891 TII.get(SPIRV::OpGroupNonUniformShuffle)) 1892 .addDef(ResVReg) 1893 .addUse(GR.getSPIRVTypeID(ResType)) 1894 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)) 1895 .addUse(I.getOperand(2).getReg()) 1896 .addUse(I.getOperand(3).getReg()); 1897 } 1898 1899 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1900 const SPIRVType *ResType, 1901 MachineInstr &I) const { 1902 MachineBasicBlock &BB = *I.getParent(); 1903 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1904 .addDef(ResVReg) 1905 .addUse(GR.getSPIRVTypeID(ResType)) 1906 .addUse(I.getOperand(1).getReg()) 1907 .constrainAllUses(TII, TRI, RBI); 1908 } 1909 1910 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1911 const SPIRVType *ResType, 1912 MachineInstr &I) const { 1913 // There is no way to implement `freeze` correctly without support on SPIR-V 1914 // standard side, but we may at least address a simple (static) case when 1915 // undef/poison value presence is obvious. The main benefit of even 1916 // incomplete `freeze` support is preventing of translation from crashing due 1917 // to lack of support on legalization and instruction selection steps. 1918 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1919 return false; 1920 Register OpReg = I.getOperand(1).getReg(); 1921 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1922 Register Reg; 1923 switch (Def->getOpcode()) { 1924 case SPIRV::ASSIGN_TYPE: 1925 if (MachineInstr *AssignToDef = 1926 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1927 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1928 Reg = Def->getOperand(2).getReg(); 1929 } 1930 break; 1931 case SPIRV::OpUndef: 1932 Reg = Def->getOperand(1).getReg(); 1933 break; 1934 } 1935 unsigned DestOpCode; 1936 if (Reg.isValid()) { 1937 DestOpCode = SPIRV::OpConstantNull; 1938 } else { 1939 DestOpCode = TargetOpcode::COPY; 1940 Reg = OpReg; 1941 } 1942 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1943 .addDef(I.getOperand(0).getReg()) 1944 .addUse(Reg) 1945 .constrainAllUses(TII, TRI, RBI); 1946 } 1947 return false; 1948 } 1949 1950 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1951 const SPIRVType *ResType) { 1952 Register OpReg = ResType->getOperand(2).getReg(); 1953 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1954 if (!OpDef) 1955 return 0; 1956 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1957 OpDef->getOperand(1).isReg()) { 1958 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1959 OpDef = RefDef; 1960 } 1961 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 1962 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 1963 : 0; 1964 return N; 1965 } 1966 1967 // Return true if the type represents a constant register 1968 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, 1969 SmallPtrSet<SPIRVType *, 4> &Visited) { 1970 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1971 OpDef->getOperand(1).isReg()) { 1972 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1973 OpDef = RefDef; 1974 } 1975 1976 if (Visited.contains(OpDef)) 1977 return true; 1978 Visited.insert(OpDef); 1979 1980 unsigned Opcode = OpDef->getOpcode(); 1981 switch (Opcode) { 1982 case TargetOpcode::G_CONSTANT: 1983 case TargetOpcode::G_FCONSTANT: 1984 return true; 1985 case TargetOpcode::G_INTRINSIC: 1986 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 1987 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 1988 return cast<GIntrinsic>(*OpDef).getIntrinsicID() == 1989 Intrinsic::spv_const_composite; 1990 case TargetOpcode::G_BUILD_VECTOR: 1991 case TargetOpcode::G_SPLAT_VECTOR: { 1992 for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); 1993 i++) { 1994 SPIRVType *OpNestedDef = 1995 OpDef->getOperand(i).isReg() 1996 ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) 1997 : nullptr; 1998 if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) 1999 return false; 2000 } 2001 return true; 2002 } 2003 } 2004 return false; 2005 } 2006 2007 // Return true if the virtual register represents a constant 2008 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 2009 SmallPtrSet<SPIRVType *, 4> Visited; 2010 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 2011 return isConstReg(MRI, OpDef, Visited); 2012 return false; 2013 } 2014 2015 bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, 2016 const SPIRVType *ResType, 2017 MachineInstr &I) const { 2018 unsigned N = 0; 2019 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2020 N = GR.getScalarOrVectorComponentCount(ResType); 2021 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2022 N = getArrayComponentCount(MRI, ResType); 2023 else 2024 report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); 2025 if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) 2026 report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); 2027 2028 // check if we may construct a constant vector 2029 bool IsConst = true; 2030 for (unsigned i = I.getNumExplicitDefs(); 2031 i < I.getNumExplicitOperands() && IsConst; ++i) 2032 if (!isConstReg(MRI, I.getOperand(i).getReg())) 2033 IsConst = false; 2034 2035 if (!IsConst && N < 2) 2036 report_fatal_error( 2037 "There must be at least two constituent operands in a vector"); 2038 2039 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2040 TII.get(IsConst ? SPIRV::OpConstantComposite 2041 : SPIRV::OpCompositeConstruct)) 2042 .addDef(ResVReg) 2043 .addUse(GR.getSPIRVTypeID(ResType)); 2044 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 2045 MIB.addUse(I.getOperand(i).getReg()); 2046 return MIB.constrainAllUses(TII, TRI, RBI); 2047 } 2048 2049 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 2050 const SPIRVType *ResType, 2051 MachineInstr &I) const { 2052 unsigned N = 0; 2053 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2054 N = GR.getScalarOrVectorComponentCount(ResType); 2055 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 2056 N = getArrayComponentCount(MRI, ResType); 2057 else 2058 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 2059 2060 unsigned OpIdx = I.getNumExplicitDefs(); 2061 if (!I.getOperand(OpIdx).isReg()) 2062 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 2063 2064 // check if we may construct a constant vector 2065 Register OpReg = I.getOperand(OpIdx).getReg(); 2066 bool IsConst = isConstReg(MRI, OpReg); 2067 2068 if (!IsConst && N < 2) 2069 report_fatal_error( 2070 "There must be at least two constituent operands in a vector"); 2071 2072 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 2073 TII.get(IsConst ? SPIRV::OpConstantComposite 2074 : SPIRV::OpCompositeConstruct)) 2075 .addDef(ResVReg) 2076 .addUse(GR.getSPIRVTypeID(ResType)); 2077 for (unsigned i = 0; i < N; ++i) 2078 MIB.addUse(OpReg); 2079 return MIB.constrainAllUses(TII, TRI, RBI); 2080 } 2081 2082 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 2083 const SPIRVType *ResType, 2084 unsigned CmpOpc, 2085 MachineInstr &I) const { 2086 Register Cmp0 = I.getOperand(2).getReg(); 2087 Register Cmp1 = I.getOperand(3).getReg(); 2088 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 2089 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 2090 "CMP operands should have the same type"); 2091 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 2092 .addDef(ResVReg) 2093 .addUse(GR.getSPIRVTypeID(ResType)) 2094 .addUse(Cmp0) 2095 .addUse(Cmp1) 2096 .constrainAllUses(TII, TRI, RBI); 2097 } 2098 2099 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 2100 const SPIRVType *ResType, 2101 MachineInstr &I) const { 2102 auto Pred = I.getOperand(1).getPredicate(); 2103 unsigned CmpOpc; 2104 2105 Register CmpOperand = I.getOperand(2).getReg(); 2106 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 2107 CmpOpc = getPtrCmpOpcode(Pred); 2108 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 2109 CmpOpc = getBoolCmpOpcode(Pred); 2110 else 2111 CmpOpc = getICmpOpcode(Pred); 2112 return selectCmp(ResVReg, ResType, CmpOpc, I); 2113 } 2114 2115 void SPIRVInstructionSelector::renderFImm64(MachineInstrBuilder &MIB, 2116 const MachineInstr &I, 2117 int OpIdx) const { 2118 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 2119 "Expected G_FCONSTANT"); 2120 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 2121 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 2122 } 2123 2124 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 2125 const MachineInstr &I, 2126 int OpIdx) const { 2127 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2128 "Expected G_CONSTANT"); 2129 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 2130 } 2131 2132 Register 2133 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 2134 const SPIRVType *ResType) const { 2135 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 2136 const SPIRVType *SpvI32Ty = 2137 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 2138 // Find a constant in DT or build a new one. 2139 auto ConstInt = ConstantInt::get(LLVMTy, Val); 2140 Register NewReg = GR.find(ConstInt, GR.CurMF); 2141 if (!NewReg.isValid()) { 2142 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2143 GR.add(ConstInt, GR.CurMF, NewReg); 2144 MachineInstr *MI; 2145 MachineBasicBlock &BB = *I.getParent(); 2146 if (Val == 0) { 2147 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2148 .addDef(NewReg) 2149 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 2150 } else { 2151 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2152 .addDef(NewReg) 2153 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 2154 .addImm(APInt(32, Val).getZExtValue()); 2155 } 2156 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 2157 } 2158 return NewReg; 2159 } 2160 2161 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 2162 const SPIRVType *ResType, 2163 MachineInstr &I) const { 2164 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 2165 return selectCmp(ResVReg, ResType, CmpOp, I); 2166 } 2167 2168 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 2169 MachineInstr &I) const { 2170 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2171 bool ZeroAsNull = STI.isOpenCLEnv(); 2172 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2173 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 2174 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 2175 } 2176 2177 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 2178 MachineInstr &I) const { 2179 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2180 bool ZeroAsNull = STI.isOpenCLEnv(); 2181 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 2182 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2183 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 2184 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 2185 } 2186 2187 Register SPIRVInstructionSelector::buildOnesValF(const SPIRVType *ResType, 2188 MachineInstr &I) const { 2189 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 2190 bool ZeroAsNull = STI.isOpenCLEnv(); 2191 APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); 2192 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2193 return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); 2194 return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); 2195 } 2196 2197 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 2198 const SPIRVType *ResType, 2199 MachineInstr &I) const { 2200 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2201 APInt One = 2202 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 2203 if (ResType->getOpcode() == SPIRV::OpTypeVector) 2204 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 2205 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 2206 } 2207 2208 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 2209 const SPIRVType *ResType, 2210 MachineInstr &I, 2211 bool IsSigned) const { 2212 // To extend a bool, we need to use OpSelect between constants. 2213 Register ZeroReg = buildZerosVal(ResType, I); 2214 Register OneReg = buildOnesVal(IsSigned, ResType, I); 2215 bool IsScalarBool = 2216 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 2217 unsigned Opcode = 2218 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 2219 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2220 .addDef(ResVReg) 2221 .addUse(GR.getSPIRVTypeID(ResType)) 2222 .addUse(I.getOperand(1).getReg()) 2223 .addUse(OneReg) 2224 .addUse(ZeroReg) 2225 .constrainAllUses(TII, TRI, RBI); 2226 } 2227 2228 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 2229 const SPIRVType *ResType, 2230 MachineInstr &I, bool IsSigned, 2231 unsigned Opcode) const { 2232 Register SrcReg = I.getOperand(1).getReg(); 2233 // We can convert bool value directly to float type without OpConvert*ToF, 2234 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 2235 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 2236 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 2237 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 2238 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 2239 const unsigned NumElts = ResType->getOperand(2).getImm(); 2240 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 2241 } 2242 SrcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2243 selectSelect(SrcReg, TmpType, I, false); 2244 } 2245 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 2246 } 2247 2248 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 2249 const SPIRVType *ResType, 2250 MachineInstr &I, bool IsSigned) const { 2251 Register SrcReg = I.getOperand(1).getReg(); 2252 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) 2253 return selectSelect(ResVReg, ResType, I, IsSigned); 2254 2255 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); 2256 if (SrcType == ResType) { 2257 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2258 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); 2259 if (DstRC != SrcRC && SrcRC) 2260 MRI->setRegClass(ResVReg, SrcRC); 2261 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2262 TII.get(TargetOpcode::COPY)) 2263 .addDef(ResVReg) 2264 .addUse(SrcReg) 2265 .constrainAllUses(TII, TRI, RBI); 2266 } 2267 2268 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2269 return selectUnOp(ResVReg, ResType, I, Opcode); 2270 } 2271 2272 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 2273 Register ResVReg, 2274 MachineInstr &I, 2275 const SPIRVType *IntTy, 2276 const SPIRVType *BoolTy) const { 2277 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 2278 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2279 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 2280 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 2281 Register Zero = buildZerosVal(IntTy, I); 2282 Register One = buildOnesVal(false, IntTy, I); 2283 MachineBasicBlock &BB = *I.getParent(); 2284 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2285 .addDef(BitIntReg) 2286 .addUse(GR.getSPIRVTypeID(IntTy)) 2287 .addUse(IntReg) 2288 .addUse(One) 2289 .constrainAllUses(TII, TRI, RBI); 2290 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 2291 .addDef(ResVReg) 2292 .addUse(GR.getSPIRVTypeID(BoolTy)) 2293 .addUse(BitIntReg) 2294 .addUse(Zero) 2295 .constrainAllUses(TII, TRI, RBI); 2296 } 2297 2298 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 2299 const SPIRVType *ResType, 2300 MachineInstr &I) const { 2301 Register IntReg = I.getOperand(1).getReg(); 2302 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 2303 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) 2304 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 2305 if (ArgType == ResType) { 2306 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(ResVReg); 2307 const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(IntReg); 2308 if (DstRC != SrcRC && SrcRC) 2309 MRI->setRegClass(ResVReg, SrcRC); 2310 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 2311 TII.get(TargetOpcode::COPY)) 2312 .addDef(ResVReg) 2313 .addUse(IntReg) 2314 .constrainAllUses(TII, TRI, RBI); 2315 } 2316 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 2317 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 2318 return selectUnOp(ResVReg, ResType, I, Opcode); 2319 } 2320 2321 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 2322 const SPIRVType *ResType, 2323 const APInt &Imm, 2324 MachineInstr &I) const { 2325 unsigned TyOpcode = ResType->getOpcode(); 2326 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 2327 MachineBasicBlock &BB = *I.getParent(); 2328 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 2329 Imm.isZero()) 2330 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2331 .addDef(ResVReg) 2332 .addUse(GR.getSPIRVTypeID(ResType)) 2333 .constrainAllUses(TII, TRI, RBI); 2334 if (TyOpcode == SPIRV::OpTypeInt) { 2335 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 2336 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 2337 if (Reg == ResVReg) 2338 return true; 2339 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2340 .addDef(ResVReg) 2341 .addUse(Reg) 2342 .constrainAllUses(TII, TRI, RBI); 2343 } 2344 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 2345 .addDef(ResVReg) 2346 .addUse(GR.getSPIRVTypeID(ResType)); 2347 // <=32-bit integers should be caught by the sdag pattern. 2348 assert(Imm.getBitWidth() > 32); 2349 addNumImm(Imm, MIB); 2350 return MIB.constrainAllUses(TII, TRI, RBI); 2351 } 2352 2353 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 2354 const SPIRVType *ResType, 2355 MachineInstr &I) const { 2356 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2357 .addDef(ResVReg) 2358 .addUse(GR.getSPIRVTypeID(ResType)) 2359 .constrainAllUses(TII, TRI, RBI); 2360 } 2361 2362 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2363 assert(MO.isReg()); 2364 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2365 if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { 2366 assert(TypeInst->getOperand(1).isReg()); 2367 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2368 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 2369 } 2370 return TypeInst->getOpcode() == SPIRV::OpConstantI; 2371 } 2372 2373 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 2374 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 2375 if (TypeInst->getOpcode() == SPIRV::OpConstantI) 2376 return TypeInst->getOperand(2).getImm(); 2377 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 2378 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 2379 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 2380 } 2381 2382 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 2383 const SPIRVType *ResType, 2384 MachineInstr &I) const { 2385 MachineBasicBlock &BB = *I.getParent(); 2386 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 2387 .addDef(ResVReg) 2388 .addUse(GR.getSPIRVTypeID(ResType)) 2389 // object to insert 2390 .addUse(I.getOperand(3).getReg()) 2391 // composite to insert into 2392 .addUse(I.getOperand(2).getReg()); 2393 for (unsigned i = 4; i < I.getNumOperands(); i++) 2394 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2395 return MIB.constrainAllUses(TII, TRI, RBI); 2396 } 2397 2398 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 2399 const SPIRVType *ResType, 2400 MachineInstr &I) const { 2401 MachineBasicBlock &BB = *I.getParent(); 2402 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2403 .addDef(ResVReg) 2404 .addUse(GR.getSPIRVTypeID(ResType)) 2405 .addUse(I.getOperand(2).getReg()); 2406 for (unsigned i = 3; i < I.getNumOperands(); i++) 2407 MIB.addImm(foldImm(I.getOperand(i), MRI)); 2408 return MIB.constrainAllUses(TII, TRI, RBI); 2409 } 2410 2411 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 2412 const SPIRVType *ResType, 2413 MachineInstr &I) const { 2414 if (isImm(I.getOperand(4), MRI)) 2415 return selectInsertVal(ResVReg, ResType, I); 2416 MachineBasicBlock &BB = *I.getParent(); 2417 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 2418 .addDef(ResVReg) 2419 .addUse(GR.getSPIRVTypeID(ResType)) 2420 .addUse(I.getOperand(2).getReg()) 2421 .addUse(I.getOperand(3).getReg()) 2422 .addUse(I.getOperand(4).getReg()) 2423 .constrainAllUses(TII, TRI, RBI); 2424 } 2425 2426 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 2427 const SPIRVType *ResType, 2428 MachineInstr &I) const { 2429 if (isImm(I.getOperand(3), MRI)) 2430 return selectExtractVal(ResVReg, ResType, I); 2431 MachineBasicBlock &BB = *I.getParent(); 2432 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 2433 .addDef(ResVReg) 2434 .addUse(GR.getSPIRVTypeID(ResType)) 2435 .addUse(I.getOperand(2).getReg()) 2436 .addUse(I.getOperand(3).getReg()) 2437 .constrainAllUses(TII, TRI, RBI); 2438 } 2439 2440 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 2441 const SPIRVType *ResType, 2442 MachineInstr &I) const { 2443 const bool IsGEPInBounds = I.getOperand(2).getImm(); 2444 2445 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 2446 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 2447 // we have to use Op[InBounds]AccessChain. 2448 const unsigned Opcode = STI.isVulkanEnv() 2449 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 2450 : SPIRV::OpAccessChain) 2451 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 2452 : SPIRV::OpPtrAccessChain); 2453 2454 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 2455 .addDef(ResVReg) 2456 .addUse(GR.getSPIRVTypeID(ResType)) 2457 // Object to get a pointer to. 2458 .addUse(I.getOperand(3).getReg()); 2459 // Adding indices. 2460 const unsigned StartingIndex = 2461 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 2462 ? 5 2463 : 4; 2464 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 2465 Res.addUse(I.getOperand(i).getReg()); 2466 return Res.constrainAllUses(TII, TRI, RBI); 2467 } 2468 2469 // Maybe wrap a value into OpSpecConstantOp 2470 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 2471 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 2472 bool Result = true; 2473 unsigned Lim = I.getNumExplicitOperands(); 2474 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 2475 Register OpReg = I.getOperand(i).getReg(); 2476 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 2477 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 2478 SmallPtrSet<SPIRVType *, 4> Visited; 2479 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || 2480 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || 2481 GR.isAggregateType(OpType)) { 2482 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 2483 // by selectAddrSpaceCast() 2484 CompositeArgs.push_back(OpReg); 2485 continue; 2486 } 2487 MachineFunction *MF = I.getMF(); 2488 Register WrapReg = GR.find(OpDefine, MF); 2489 if (WrapReg.isValid()) { 2490 CompositeArgs.push_back(WrapReg); 2491 continue; 2492 } 2493 // Create a new register for the wrapper 2494 WrapReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2495 GR.add(OpDefine, MF, WrapReg); 2496 CompositeArgs.push_back(WrapReg); 2497 // Decorate the wrapper register and generate a new instruction 2498 MRI->setType(WrapReg, LLT::pointer(0, 64)); 2499 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 2500 MachineBasicBlock &BB = *I.getParent(); 2501 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 2502 .addDef(WrapReg) 2503 .addUse(GR.getSPIRVTypeID(OpType)) 2504 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 2505 .addUse(OpReg) 2506 .constrainAllUses(TII, TRI, RBI); 2507 if (!Result) 2508 break; 2509 } 2510 return Result; 2511 } 2512 2513 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 2514 const SPIRVType *ResType, 2515 MachineInstr &I) const { 2516 MachineBasicBlock &BB = *I.getParent(); 2517 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 2518 switch (IID) { 2519 case Intrinsic::spv_load: 2520 return selectLoad(ResVReg, ResType, I); 2521 case Intrinsic::spv_store: 2522 return selectStore(I); 2523 case Intrinsic::spv_extractv: 2524 return selectExtractVal(ResVReg, ResType, I); 2525 case Intrinsic::spv_insertv: 2526 return selectInsertVal(ResVReg, ResType, I); 2527 case Intrinsic::spv_extractelt: 2528 return selectExtractElt(ResVReg, ResType, I); 2529 case Intrinsic::spv_insertelt: 2530 return selectInsertElt(ResVReg, ResType, I); 2531 case Intrinsic::spv_gep: 2532 return selectGEP(ResVReg, ResType, I); 2533 case Intrinsic::spv_unref_global: 2534 case Intrinsic::spv_init_global: { 2535 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 2536 MachineInstr *Init = I.getNumExplicitOperands() > 2 2537 ? MRI->getVRegDef(I.getOperand(2).getReg()) 2538 : nullptr; 2539 assert(MI); 2540 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 2541 } 2542 case Intrinsic::spv_undef: { 2543 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 2544 .addDef(ResVReg) 2545 .addUse(GR.getSPIRVTypeID(ResType)); 2546 return MIB.constrainAllUses(TII, TRI, RBI); 2547 } 2548 case Intrinsic::spv_const_composite: { 2549 // If no values are attached, the composite is null constant. 2550 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 2551 // Select a proper instruction. 2552 unsigned Opcode = SPIRV::OpConstantNull; 2553 SmallVector<Register> CompositeArgs; 2554 if (!IsNull) { 2555 Opcode = SPIRV::OpConstantComposite; 2556 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 2557 return false; 2558 } 2559 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2560 .addDef(ResVReg) 2561 .addUse(GR.getSPIRVTypeID(ResType)); 2562 // skip type MD node we already used when generated assign.type for this 2563 if (!IsNull) { 2564 for (Register OpReg : CompositeArgs) 2565 MIB.addUse(OpReg); 2566 } 2567 return MIB.constrainAllUses(TII, TRI, RBI); 2568 } 2569 case Intrinsic::spv_assign_name: { 2570 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 2571 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 2572 for (unsigned i = I.getNumExplicitDefs() + 2; 2573 i < I.getNumExplicitOperands(); ++i) { 2574 MIB.addImm(I.getOperand(i).getImm()); 2575 } 2576 return MIB.constrainAllUses(TII, TRI, RBI); 2577 } 2578 case Intrinsic::spv_switch: { 2579 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 2580 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2581 if (I.getOperand(i).isReg()) 2582 MIB.addReg(I.getOperand(i).getReg()); 2583 else if (I.getOperand(i).isCImm()) 2584 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 2585 else if (I.getOperand(i).isMBB()) 2586 MIB.addMBB(I.getOperand(i).getMBB()); 2587 else 2588 llvm_unreachable("Unexpected OpSwitch operand"); 2589 } 2590 return MIB.constrainAllUses(TII, TRI, RBI); 2591 } 2592 case Intrinsic::spv_loop_merge: 2593 case Intrinsic::spv_selection_merge: { 2594 const auto Opcode = IID == Intrinsic::spv_selection_merge 2595 ? SPIRV::OpSelectionMerge 2596 : SPIRV::OpLoopMerge; 2597 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)); 2598 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 2599 assert(I.getOperand(i).isMBB()); 2600 MIB.addMBB(I.getOperand(i).getMBB()); 2601 } 2602 MIB.addImm(SPIRV::SelectionControl::None); 2603 return MIB.constrainAllUses(TII, TRI, RBI); 2604 } 2605 case Intrinsic::spv_cmpxchg: 2606 return selectAtomicCmpXchg(ResVReg, ResType, I); 2607 case Intrinsic::spv_unreachable: 2608 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 2609 break; 2610 case Intrinsic::spv_alloca: 2611 return selectFrameIndex(ResVReg, ResType, I); 2612 case Intrinsic::spv_alloca_array: 2613 return selectAllocaArray(ResVReg, ResType, I); 2614 case Intrinsic::spv_assume: 2615 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2616 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 2617 .addUse(I.getOperand(1).getReg()); 2618 break; 2619 case Intrinsic::spv_expect: 2620 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 2621 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 2622 .addDef(ResVReg) 2623 .addUse(GR.getSPIRVTypeID(ResType)) 2624 .addUse(I.getOperand(2).getReg()) 2625 .addUse(I.getOperand(3).getReg()); 2626 break; 2627 case Intrinsic::arithmetic_fence: 2628 if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) 2629 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT)) 2630 .addDef(ResVReg) 2631 .addUse(GR.getSPIRVTypeID(ResType)) 2632 .addUse(I.getOperand(2).getReg()); 2633 else 2634 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 2635 .addUse(I.getOperand(2).getReg()); 2636 break; 2637 case Intrinsic::spv_thread_id: 2638 return selectSpvThreadId(ResVReg, ResType, I); 2639 case Intrinsic::spv_fdot: 2640 return selectFloatDot(ResVReg, ResType, I); 2641 case Intrinsic::spv_udot: 2642 case Intrinsic::spv_sdot: 2643 return selectIntegerDot(ResVReg, ResType, I); 2644 case Intrinsic::spv_dot4add_i8packed: 2645 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || 2646 STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) 2647 return selectDot4AddPacked<true>(ResVReg, ResType, I); 2648 return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); 2649 case Intrinsic::spv_all: 2650 return selectAll(ResVReg, ResType, I); 2651 case Intrinsic::spv_any: 2652 return selectAny(ResVReg, ResType, I); 2653 case Intrinsic::spv_cross: 2654 return selectExtInst(ResVReg, ResType, I, CL::cross, GL::Cross); 2655 case Intrinsic::spv_lerp: 2656 return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); 2657 case Intrinsic::spv_length: 2658 return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); 2659 case Intrinsic::spv_degrees: 2660 return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); 2661 case Intrinsic::spv_frac: 2662 return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); 2663 case Intrinsic::spv_normalize: 2664 return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); 2665 case Intrinsic::spv_rsqrt: 2666 return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); 2667 case Intrinsic::spv_sign: 2668 return selectSign(ResVReg, ResType, I); 2669 case Intrinsic::spv_group_memory_barrier_with_group_sync: { 2670 Register MemSemReg = 2671 buildI32Constant(SPIRV::MemorySemantics::SequentiallyConsistent, I); 2672 Register ScopeReg = buildI32Constant(SPIRV::Scope::Workgroup, I); 2673 MachineBasicBlock &BB = *I.getParent(); 2674 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpControlBarrier)) 2675 .addUse(ScopeReg) 2676 .addUse(ScopeReg) 2677 .addUse(MemSemReg) 2678 .constrainAllUses(TII, TRI, RBI); 2679 } break; 2680 case Intrinsic::spv_lifetime_start: 2681 case Intrinsic::spv_lifetime_end: { 2682 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 2683 : SPIRV::OpLifetimeStop; 2684 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 2685 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 2686 if (Size == -1) 2687 Size = 0; 2688 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 2689 } break; 2690 case Intrinsic::spv_saturate: 2691 return selectSaturate(ResVReg, ResType, I); 2692 case Intrinsic::spv_wave_is_first_lane: { 2693 SPIRVType *IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); 2694 return BuildMI(BB, I, I.getDebugLoc(), 2695 TII.get(SPIRV::OpGroupNonUniformElect)) 2696 .addDef(ResVReg) 2697 .addUse(GR.getSPIRVTypeID(ResType)) 2698 .addUse(GR.getOrCreateConstInt(3, I, IntTy, TII)); 2699 } 2700 case Intrinsic::spv_wave_readlane: 2701 return selectWaveReadLaneAt(ResVReg, ResType, I); 2702 case Intrinsic::spv_step: 2703 return selectExtInst(ResVReg, ResType, I, CL::step, GL::Step); 2704 case Intrinsic::spv_radians: 2705 return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); 2706 // Discard intrinsics which we do not expect to actually represent code after 2707 // lowering or intrinsics which are not implemented but should not crash when 2708 // found in a customer's LLVM IR input. 2709 case Intrinsic::instrprof_increment: 2710 case Intrinsic::instrprof_increment_step: 2711 case Intrinsic::instrprof_value_profile: 2712 break; 2713 // Discard internal intrinsics. 2714 case Intrinsic::spv_value_md: 2715 break; 2716 case Intrinsic::spv_handle_fromBinding: { 2717 selectHandleFromBinding(ResVReg, ResType, I); 2718 return true; 2719 } 2720 default: { 2721 std::string DiagMsg; 2722 raw_string_ostream OS(DiagMsg); 2723 I.print(OS); 2724 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 2725 report_fatal_error(DiagMsg.c_str(), false); 2726 } 2727 } 2728 return true; 2729 } 2730 2731 void SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, 2732 const SPIRVType *ResType, 2733 MachineInstr &I) const { 2734 2735 uint32_t Set = foldImm(I.getOperand(2), MRI); 2736 uint32_t Binding = foldImm(I.getOperand(3), MRI); 2737 uint32_t ArraySize = foldImm(I.getOperand(4), MRI); 2738 Register IndexReg = I.getOperand(5).getReg(); 2739 bool IsNonUniform = ArraySize > 1 && foldImm(I.getOperand(6), MRI); 2740 2741 MachineIRBuilder MIRBuilder(I); 2742 Register VarReg = buildPointerToResource(ResType, Set, Binding, ArraySize, 2743 IndexReg, IsNonUniform, MIRBuilder); 2744 2745 if (IsNonUniform) 2746 buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); 2747 2748 // TODO: For now we assume the resource is an image, which needs to be 2749 // loaded to get the handle. That will not be true for storage buffers. 2750 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2751 .addDef(ResVReg) 2752 .addUse(GR.getSPIRVTypeID(ResType)) 2753 .addUse(VarReg); 2754 } 2755 2756 Register SPIRVInstructionSelector::buildPointerToResource( 2757 const SPIRVType *ResType, uint32_t Set, uint32_t Binding, 2758 uint32_t ArraySize, Register IndexReg, bool IsNonUniform, 2759 MachineIRBuilder MIRBuilder) const { 2760 if (ArraySize == 1) 2761 return GR.getOrCreateGlobalVariableWithBinding(ResType, Set, Binding, 2762 MIRBuilder); 2763 2764 const SPIRVType *VarType = GR.getOrCreateSPIRVArrayType( 2765 ResType, ArraySize, *MIRBuilder.getInsertPt(), TII); 2766 Register VarReg = GR.getOrCreateGlobalVariableWithBinding( 2767 VarType, Set, Binding, MIRBuilder); 2768 2769 SPIRVType *ResPointerType = GR.getOrCreateSPIRVPointerType( 2770 ResType, MIRBuilder, SPIRV::StorageClass::UniformConstant); 2771 2772 Register AcReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 2773 if (IsNonUniform) { 2774 // It is unclear which value needs to be marked an non-uniform, so both 2775 // the index and the access changed are decorated as non-uniform. 2776 buildOpDecorate(IndexReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2777 buildOpDecorate(AcReg, MIRBuilder, SPIRV::Decoration::NonUniformEXT, {}); 2778 } 2779 2780 MIRBuilder.buildInstr(SPIRV::OpAccessChain) 2781 .addDef(AcReg) 2782 .addUse(GR.getSPIRVTypeID(ResPointerType)) 2783 .addUse(VarReg) 2784 .addUse(IndexReg); 2785 2786 return AcReg; 2787 } 2788 2789 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 2790 const SPIRVType *ResType, 2791 MachineInstr &I) const { 2792 // there was an allocation size parameter to the allocation instruction 2793 // that is not 1 2794 MachineBasicBlock &BB = *I.getParent(); 2795 return BuildMI(BB, I, I.getDebugLoc(), 2796 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 2797 .addDef(ResVReg) 2798 .addUse(GR.getSPIRVTypeID(ResType)) 2799 .addUse(I.getOperand(2).getReg()) 2800 .constrainAllUses(TII, TRI, RBI); 2801 } 2802 2803 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 2804 const SPIRVType *ResType, 2805 MachineInstr &I) const { 2806 // Change order of instructions if needed: all OpVariable instructions in a 2807 // function must be the first instructions in the first block 2808 MachineFunction *MF = I.getParent()->getParent(); 2809 MachineBasicBlock *MBB = &MF->front(); 2810 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 2811 bool IsHeader = false; 2812 unsigned Opcode; 2813 for (; It != E && It != I; ++It) { 2814 Opcode = It->getOpcode(); 2815 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 2816 IsHeader = true; 2817 } else if (IsHeader && 2818 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 2819 ++It; 2820 break; 2821 } 2822 } 2823 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 2824 .addDef(ResVReg) 2825 .addUse(GR.getSPIRVTypeID(ResType)) 2826 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 2827 .constrainAllUses(TII, TRI, RBI); 2828 } 2829 2830 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 2831 // InstructionSelector walks backwards through the instructions. We can use 2832 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 2833 // first, so can generate an OpBranchConditional here. If there is no 2834 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 2835 const MachineInstr *PrevI = I.getPrevNode(); 2836 MachineBasicBlock &MBB = *I.getParent(); 2837 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 2838 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2839 .addUse(PrevI->getOperand(0).getReg()) 2840 .addMBB(PrevI->getOperand(1).getMBB()) 2841 .addMBB(I.getOperand(0).getMBB()) 2842 .constrainAllUses(TII, TRI, RBI); 2843 } 2844 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 2845 .addMBB(I.getOperand(0).getMBB()) 2846 .constrainAllUses(TII, TRI, RBI); 2847 } 2848 2849 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 2850 // InstructionSelector walks backwards through the instructions. For an 2851 // explicit conditional branch with no fallthrough, we use both a G_BR and a 2852 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 2853 // generate the OpBranchConditional in selectBranch above. 2854 // 2855 // If an OpBranchConditional has been generated, we simply return, as the work 2856 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 2857 // implicit fallthrough to the next basic block, so we need to create an 2858 // OpBranchConditional with an explicit "false" argument pointing to the next 2859 // basic block that LLVM would fall through to. 2860 const MachineInstr *NextI = I.getNextNode(); 2861 // Check if this has already been successfully selected. 2862 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 2863 return true; 2864 // Must be relying on implicit block fallthrough, so generate an 2865 // OpBranchConditional with the "next" basic block as the "false" target. 2866 MachineBasicBlock &MBB = *I.getParent(); 2867 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 2868 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 2869 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2870 .addUse(I.getOperand(0).getReg()) 2871 .addMBB(I.getOperand(1).getMBB()) 2872 .addMBB(NextMBB) 2873 .constrainAllUses(TII, TRI, RBI); 2874 } 2875 2876 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 2877 const SPIRVType *ResType, 2878 MachineInstr &I) const { 2879 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 2880 .addDef(ResVReg) 2881 .addUse(GR.getSPIRVTypeID(ResType)); 2882 const unsigned NumOps = I.getNumOperands(); 2883 for (unsigned i = 1; i < NumOps; i += 2) { 2884 MIB.addUse(I.getOperand(i + 0).getReg()); 2885 MIB.addMBB(I.getOperand(i + 1).getMBB()); 2886 } 2887 return MIB.constrainAllUses(TII, TRI, RBI); 2888 } 2889 2890 bool SPIRVInstructionSelector::selectGlobalValue( 2891 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 2892 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 2893 MachineIRBuilder MIRBuilder(I); 2894 const GlobalValue *GV = I.getOperand(1).getGlobal(); 2895 Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); 2896 SPIRVType *PointerBaseType; 2897 if (GVType->isArrayTy()) { 2898 SPIRVType *ArrayElementType = 2899 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 2900 SPIRV::AccessQualifier::ReadWrite, false); 2901 PointerBaseType = GR.getOrCreateSPIRVArrayType( 2902 ArrayElementType, GVType->getArrayNumElements(), I, TII); 2903 } else { 2904 PointerBaseType = GR.getOrCreateSPIRVType( 2905 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 2906 } 2907 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 2908 PointerBaseType, I, TII, 2909 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 2910 2911 std::string GlobalIdent; 2912 if (!GV->hasName()) { 2913 unsigned &ID = UnnamedGlobalIDs[GV]; 2914 if (ID == 0) 2915 ID = UnnamedGlobalIDs.size(); 2916 GlobalIdent = "__unnamed_" + Twine(ID).str(); 2917 } else { 2918 GlobalIdent = GV->getGlobalIdentifier(); 2919 } 2920 2921 // Behaviour of functions as operands depends on availability of the 2922 // corresponding extension (SPV_INTEL_function_pointers): 2923 // - If there is an extension to operate with functions as operands: 2924 // We create a proper constant operand and evaluate a correct type for a 2925 // function pointer. 2926 // - Without the required extension: 2927 // We have functions as operands in tests with blocks of instruction e.g. in 2928 // transcoding/global_block.ll. These operands are not used and should be 2929 // substituted by zero constants. Their type is expected to be always 2930 // OpTypePointer Function %uchar. 2931 if (isa<Function>(GV)) { 2932 const Constant *ConstVal = GV; 2933 MachineBasicBlock &BB = *I.getParent(); 2934 Register NewReg = GR.find(ConstVal, GR.CurMF); 2935 if (!NewReg.isValid()) { 2936 Register NewReg = ResVReg; 2937 GR.add(ConstVal, GR.CurMF, NewReg); 2938 const Function *GVFun = 2939 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 2940 ? dyn_cast<Function>(GV) 2941 : nullptr; 2942 if (GVFun) { 2943 // References to a function via function pointers generate virtual 2944 // registers without a definition. We will resolve it later, during 2945 // module analysis stage. 2946 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2947 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); 2948 MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); 2949 MachineInstrBuilder MB = 2950 BuildMI(BB, I, I.getDebugLoc(), 2951 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 2952 .addDef(NewReg) 2953 .addUse(GR.getSPIRVTypeID(ResType)) 2954 .addUse(FuncVReg); 2955 // mapping the function pointer to the used Function 2956 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 2957 return MB.constrainAllUses(TII, TRI, RBI); 2958 } 2959 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2960 .addDef(NewReg) 2961 .addUse(GR.getSPIRVTypeID(ResType)) 2962 .constrainAllUses(TII, TRI, RBI); 2963 } 2964 assert(NewReg != ResVReg); 2965 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2966 .addDef(ResVReg) 2967 .addUse(NewReg) 2968 .constrainAllUses(TII, TRI, RBI); 2969 } 2970 auto GlobalVar = cast<GlobalVariable>(GV); 2971 assert(GlobalVar->getName() != "llvm.global.annotations"); 2972 2973 bool HasInit = GlobalVar->hasInitializer() && 2974 !isa<UndefValue>(GlobalVar->getInitializer()); 2975 // Skip empty declaration for GVs with initilaizers till we get the decl with 2976 // passed initializer. 2977 if (HasInit && !Init) 2978 return true; 2979 2980 unsigned AddrSpace = GV->getAddressSpace(); 2981 SPIRV::StorageClass::StorageClass Storage = 2982 addressSpaceToStorageClass(AddrSpace, STI); 2983 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 2984 Storage != SPIRV::StorageClass::Function; 2985 SPIRV::LinkageType::LinkageType LnkType = 2986 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 2987 ? SPIRV::LinkageType::Import 2988 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 2989 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 2990 ? SPIRV::LinkageType::LinkOnceODR 2991 : SPIRV::LinkageType::Export); 2992 2993 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 2994 Storage, Init, GlobalVar->isConstant(), 2995 HasLnkTy, LnkType, MIRBuilder, true); 2996 return Reg.isValid(); 2997 } 2998 2999 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 3000 const SPIRVType *ResType, 3001 MachineInstr &I) const { 3002 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 3003 return selectExtInst(ResVReg, ResType, I, CL::log10); 3004 } 3005 3006 // There is no log10 instruction in the GLSL Extended Instruction set, so it 3007 // is implemented as: 3008 // log10(x) = log2(x) * (1 / log2(10)) 3009 // = log2(x) * 0.30103 3010 3011 MachineIRBuilder MIRBuilder(I); 3012 MachineBasicBlock &BB = *I.getParent(); 3013 3014 // Build log2(x). 3015 Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); 3016 bool Result = 3017 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 3018 .addDef(VarReg) 3019 .addUse(GR.getSPIRVTypeID(ResType)) 3020 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 3021 .addImm(GL::Log2) 3022 .add(I.getOperand(1)) 3023 .constrainAllUses(TII, TRI, RBI); 3024 3025 // Build 0.30103. 3026 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 3027 ResType->getOpcode() == SPIRV::OpTypeFloat); 3028 // TODO: Add matrix implementation once supported by the HLSL frontend. 3029 const SPIRVType *SpirvScalarType = 3030 ResType->getOpcode() == SPIRV::OpTypeVector 3031 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 3032 : ResType; 3033 Register ScaleReg = 3034 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 3035 3036 // Multiply log2(x) by 0.30103 to get log10(x) result. 3037 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 3038 ? SPIRV::OpVectorTimesScalar 3039 : SPIRV::OpFMulS; 3040 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 3041 .addDef(ResVReg) 3042 .addUse(GR.getSPIRVTypeID(ResType)) 3043 .addUse(VarReg) 3044 .addUse(ScaleReg) 3045 .constrainAllUses(TII, TRI, RBI); 3046 3047 return Result; 3048 } 3049 3050 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 3051 const SPIRVType *ResType, 3052 MachineInstr &I) const { 3053 // DX intrinsic: @llvm.dx.thread.id(i32) 3054 // ID Name Description 3055 // 93 ThreadId reads the thread ID 3056 3057 MachineIRBuilder MIRBuilder(I); 3058 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 3059 const SPIRVType *Vec3Ty = 3060 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 3061 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 3062 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 3063 3064 // Create new register for GlobalInvocationID builtin variable. 3065 Register NewRegister = 3066 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); 3067 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); 3068 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 3069 3070 // Build GlobalInvocationID global variable with the necessary decorations. 3071 Register Variable = GR.buildGlobalVariable( 3072 NewRegister, PtrType, 3073 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 3074 SPIRV::StorageClass::Input, nullptr, true, true, 3075 SPIRV::LinkageType::Import, MIRBuilder, false); 3076 3077 // Create new register for loading value. 3078 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 3079 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); 3080 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); 3081 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 3082 3083 // Load v3uint value from the global variable. 3084 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 3085 .addDef(LoadedRegister) 3086 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 3087 .addUse(Variable); 3088 3089 // Get Thread ID index. Expecting operand is a constant immediate value, 3090 // wrapped in a type assignment. 3091 assert(I.getOperand(2).isReg()); 3092 const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); 3093 3094 // Extract the thread ID from the loaded vector value. 3095 MachineBasicBlock &BB = *I.getParent(); 3096 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 3097 .addDef(ResVReg) 3098 .addUse(GR.getSPIRVTypeID(ResType)) 3099 .addUse(LoadedRegister) 3100 .addImm(ThreadId); 3101 return MIB.constrainAllUses(TII, TRI, RBI); 3102 } 3103 3104 namespace llvm { 3105 InstructionSelector * 3106 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 3107 const SPIRVSubtarget &Subtarget, 3108 const RegisterBankInfo &RBI) { 3109 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 3110 } 3111 } // namespace llvm 3112