1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/TargetOpcodes.h" 32 #include "llvm/IR/IntrinsicsSPIRV.h" 33 #include "llvm/Support/Debug.h" 34 35 namespace llvm { 36 37 class SPIRVMachineModuleInfo : public MachineModuleInfoImpl { 38 public: 39 SyncScope::ID Work_ItemSSID; 40 SyncScope::ID WorkGroupSSID; 41 SyncScope::ID DeviceSSID; 42 SyncScope::ID AllSVMDevicesSSID; 43 SyncScope::ID SubGroupSSID; 44 45 SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) { 46 LLVMContext &CTX = MMI.getModule()->getContext(); 47 Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item"); 48 WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); 49 DeviceSSID = CTX.getOrInsertSyncScopeID("device"); 50 AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices"); 51 SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group"); 52 } 53 }; 54 55 } // end namespace llvm 56 57 #define DEBUG_TYPE "spirv-isel" 58 59 using namespace llvm; 60 namespace CL = SPIRV::OpenCLExtInst; 61 namespace GL = SPIRV::GLSLExtInst; 62 63 using ExtInstList = 64 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 65 66 namespace { 67 68 #define GET_GLOBALISEL_PREDICATE_BITSET 69 #include "SPIRVGenGlobalISel.inc" 70 #undef GET_GLOBALISEL_PREDICATE_BITSET 71 72 class SPIRVInstructionSelector : public InstructionSelector { 73 const SPIRVSubtarget &STI; 74 const SPIRVInstrInfo &TII; 75 const SPIRVRegisterInfo &TRI; 76 const RegisterBankInfo &RBI; 77 SPIRVGlobalRegistry &GR; 78 MachineRegisterInfo *MRI; 79 SPIRVMachineModuleInfo *MMI = nullptr; 80 81 /// We need to keep track of the number we give to anonymous global values to 82 /// generate the same name every time when this is needed. 83 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 84 85 public: 86 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 87 const SPIRVSubtarget &ST, 88 const RegisterBankInfo &RBI); 89 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 90 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 91 BlockFrequencyInfo *BFI) override; 92 // Common selection code. Instruction-specific selection occurs in spvSelect. 93 bool select(MachineInstr &I) override; 94 static const char *getName() { return DEBUG_TYPE; } 95 96 #define GET_GLOBALISEL_PREDICATES_DECL 97 #include "SPIRVGenGlobalISel.inc" 98 #undef GET_GLOBALISEL_PREDICATES_DECL 99 100 #define GET_GLOBALISEL_TEMPORARIES_DECL 101 #include "SPIRVGenGlobalISel.inc" 102 #undef GET_GLOBALISEL_TEMPORARIES_DECL 103 104 private: 105 // tblgen-erated 'select' implementation, used as the initial selector for 106 // the patterns that don't require complex C++. 107 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 108 109 // All instruction-specific selection that didn't happen in "select()". 110 // Is basically a large Switch/Case delegating to all other select method. 111 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I) const; 113 114 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 115 const MachineInstr *Init = nullptr) const; 116 117 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 118 MachineInstr &I, Register SrcReg, 119 unsigned Opcode) const; 120 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 121 unsigned Opcode) const; 122 123 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I) const; 125 126 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 bool selectStore(MachineInstr &I) const; 129 130 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 131 MachineInstr &I) const; 132 bool selectStackRestore(MachineInstr &I) const; 133 134 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 135 136 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 137 MachineInstr &I, unsigned NewOpcode, 138 unsigned NegateOpcode = 0) const; 139 140 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 141 MachineInstr &I) const; 142 143 bool selectFence(MachineInstr &I) const; 144 145 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 148 bool selectAll(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I) const; 150 151 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 152 MachineInstr &I) const; 153 154 bool selectConstVector(Register ResVReg, const SPIRVType *ResType, 155 MachineInstr &I) const; 156 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 157 MachineInstr &I) const; 158 159 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 160 unsigned comparisonOpcode, MachineInstr &I) const; 161 162 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 163 MachineInstr &I) const; 164 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 165 MachineInstr &I) const; 166 167 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 168 int OpIdx) const; 169 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 170 int OpIdx) const; 171 172 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 173 MachineInstr &I) const; 174 175 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 176 bool IsSigned) const; 177 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 178 bool IsSigned, unsigned Opcode) const; 179 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 180 bool IsSigned) const; 181 182 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I) const; 184 185 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 186 const SPIRVType *intTy, const SPIRVType *boolTy) const; 187 188 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 191 MachineInstr &I) const; 192 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 195 MachineInstr &I) const; 196 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 199 MachineInstr &I) const; 200 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 201 MachineInstr &I) const; 202 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I) const; 204 205 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 206 MachineInstr &I) const; 207 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 208 MachineInstr &I) const; 209 210 bool selectBranch(MachineInstr &I) const; 211 bool selectBranchCond(MachineInstr &I) const; 212 213 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 214 MachineInstr &I) const; 215 216 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 217 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 218 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 219 MachineInstr &I, CL::OpenCLExtInst CLInst, 220 GL::GLSLExtInst GLInst) const; 221 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 222 MachineInstr &I, const ExtInstList &ExtInsts) const; 223 224 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I) const; 226 227 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I) const; 229 230 bool selectUnmergeValues(MachineInstr &I) const; 231 232 Register buildI32Constant(uint32_t Val, MachineInstr &I, 233 const SPIRVType *ResType = nullptr) const; 234 235 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 236 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 237 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 238 MachineInstr &I) const; 239 240 bool wrapIntoSpecConstantOp(MachineInstr &I, 241 SmallVector<Register> &CompositeArgs) const; 242 }; 243 244 } // end anonymous namespace 245 246 #define GET_GLOBALISEL_IMPL 247 #include "SPIRVGenGlobalISel.inc" 248 #undef GET_GLOBALISEL_IMPL 249 250 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 251 const SPIRVSubtarget &ST, 252 const RegisterBankInfo &RBI) 253 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 254 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 255 #define GET_GLOBALISEL_PREDICATES_INIT 256 #include "SPIRVGenGlobalISel.inc" 257 #undef GET_GLOBALISEL_PREDICATES_INIT 258 #define GET_GLOBALISEL_TEMPORARIES_INIT 259 #include "SPIRVGenGlobalISel.inc" 260 #undef GET_GLOBALISEL_TEMPORARIES_INIT 261 { 262 } 263 264 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 265 CodeGenCoverage *CoverageInfo, 266 ProfileSummaryInfo *PSI, 267 BlockFrequencyInfo *BFI) { 268 MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>(); 269 MRI = &MF.getRegInfo(); 270 GR.setCurrentFunc(MF); 271 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 272 } 273 274 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 275 276 // Defined in SPIRVLegalizerInfo.cpp. 277 extern bool isTypeFoldingSupported(unsigned Opcode); 278 279 bool SPIRVInstructionSelector::select(MachineInstr &I) { 280 assert(I.getParent() && "Instruction should be in a basic block!"); 281 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 282 283 Register Opcode = I.getOpcode(); 284 // If it's not a GMIR instruction, we've selected it already. 285 if (!isPreISelGenericOpcode(Opcode)) { 286 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 287 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); 288 if (isTypeFoldingSupported(Def->getOpcode())) { 289 bool Res = selectImpl(I, *CoverageInfo); 290 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 291 if (Res) 292 return Res; 293 } 294 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); 295 I.removeFromParent(); 296 return true; 297 } else if (I.getNumDefs() == 1) { 298 // Make all vregs 32 bits (for SPIR-V IDs). 299 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); 300 } 301 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 302 } 303 304 if (I.getNumOperands() != I.getNumExplicitOperands()) { 305 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 306 return false; 307 } 308 309 // Common code for getting return reg+type, and removing selected instr 310 // from parent occurs here. Instr-specific selection happens in spvSelect(). 311 bool HasDefs = I.getNumDefs() > 0; 312 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 313 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 314 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 315 if (spvSelect(ResVReg, ResType, I)) { 316 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). 317 for (unsigned i = 0; i < I.getNumDefs(); ++i) 318 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32)); 319 I.removeFromParent(); 320 return true; 321 } 322 return false; 323 } 324 325 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 326 const SPIRVType *ResType, 327 MachineInstr &I) const { 328 const unsigned Opcode = I.getOpcode(); 329 if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT) 330 return selectImpl(I, *CoverageInfo); 331 switch (Opcode) { 332 case TargetOpcode::G_CONSTANT: 333 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 334 I); 335 case TargetOpcode::G_GLOBAL_VALUE: 336 return selectGlobalValue(ResVReg, I); 337 case TargetOpcode::G_IMPLICIT_DEF: 338 return selectOpUndef(ResVReg, ResType, I); 339 case TargetOpcode::G_FREEZE: 340 return selectFreeze(ResVReg, ResType, I); 341 342 case TargetOpcode::G_INTRINSIC: 343 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 344 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 345 return selectIntrinsic(ResVReg, ResType, I); 346 case TargetOpcode::G_BITREVERSE: 347 return selectBitreverse(ResVReg, ResType, I); 348 349 case TargetOpcode::G_BUILD_VECTOR: 350 return selectConstVector(ResVReg, ResType, I); 351 case TargetOpcode::G_SPLAT_VECTOR: 352 return selectSplatVector(ResVReg, ResType, I); 353 354 case TargetOpcode::G_SHUFFLE_VECTOR: { 355 MachineBasicBlock &BB = *I.getParent(); 356 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 357 .addDef(ResVReg) 358 .addUse(GR.getSPIRVTypeID(ResType)) 359 .addUse(I.getOperand(1).getReg()) 360 .addUse(I.getOperand(2).getReg()); 361 for (auto V : I.getOperand(3).getShuffleMask()) 362 MIB.addImm(V); 363 return MIB.constrainAllUses(TII, TRI, RBI); 364 } 365 case TargetOpcode::G_MEMMOVE: 366 case TargetOpcode::G_MEMCPY: 367 case TargetOpcode::G_MEMSET: 368 return selectMemOperation(ResVReg, I); 369 370 case TargetOpcode::G_ICMP: 371 return selectICmp(ResVReg, ResType, I); 372 case TargetOpcode::G_FCMP: 373 return selectFCmp(ResVReg, ResType, I); 374 375 case TargetOpcode::G_FRAME_INDEX: 376 return selectFrameIndex(ResVReg, ResType, I); 377 378 case TargetOpcode::G_LOAD: 379 return selectLoad(ResVReg, ResType, I); 380 case TargetOpcode::G_STORE: 381 return selectStore(I); 382 383 case TargetOpcode::G_BR: 384 return selectBranch(I); 385 case TargetOpcode::G_BRCOND: 386 return selectBranchCond(I); 387 388 case TargetOpcode::G_PHI: 389 return selectPhi(ResVReg, ResType, I); 390 391 case TargetOpcode::G_FPTOSI: 392 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 393 case TargetOpcode::G_FPTOUI: 394 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 395 396 case TargetOpcode::G_SITOFP: 397 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 398 case TargetOpcode::G_UITOFP: 399 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 400 401 case TargetOpcode::G_CTPOP: 402 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 403 case TargetOpcode::G_SMIN: 404 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 405 case TargetOpcode::G_UMIN: 406 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 407 408 case TargetOpcode::G_SMAX: 409 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 410 case TargetOpcode::G_UMAX: 411 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 412 413 case TargetOpcode::G_FMA: 414 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 415 416 case TargetOpcode::G_FPOW: 417 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 418 case TargetOpcode::G_FPOWI: 419 return selectExtInst(ResVReg, ResType, I, CL::pown); 420 421 case TargetOpcode::G_FEXP: 422 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 423 case TargetOpcode::G_FEXP2: 424 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 425 426 case TargetOpcode::G_FLOG: 427 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 428 case TargetOpcode::G_FLOG2: 429 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 430 case TargetOpcode::G_FLOG10: 431 return selectLog10(ResVReg, ResType, I); 432 433 case TargetOpcode::G_FABS: 434 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 435 case TargetOpcode::G_ABS: 436 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 437 438 case TargetOpcode::G_FMINNUM: 439 case TargetOpcode::G_FMINIMUM: 440 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 441 case TargetOpcode::G_FMAXNUM: 442 case TargetOpcode::G_FMAXIMUM: 443 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 444 445 case TargetOpcode::G_FCOPYSIGN: 446 return selectExtInst(ResVReg, ResType, I, CL::copysign); 447 448 case TargetOpcode::G_FCEIL: 449 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 450 case TargetOpcode::G_FFLOOR: 451 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 452 453 case TargetOpcode::G_FCOS: 454 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 455 case TargetOpcode::G_FSIN: 456 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 457 458 case TargetOpcode::G_FSQRT: 459 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 460 461 case TargetOpcode::G_CTTZ: 462 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 463 return selectExtInst(ResVReg, ResType, I, CL::ctz); 464 case TargetOpcode::G_CTLZ: 465 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 466 return selectExtInst(ResVReg, ResType, I, CL::clz); 467 468 case TargetOpcode::G_INTRINSIC_ROUND: 469 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 470 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 471 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 472 case TargetOpcode::G_INTRINSIC_TRUNC: 473 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 474 case TargetOpcode::G_FRINT: 475 case TargetOpcode::G_FNEARBYINT: 476 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 477 478 case TargetOpcode::G_SMULH: 479 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 480 case TargetOpcode::G_UMULH: 481 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 482 483 case TargetOpcode::G_SEXT: 484 return selectExt(ResVReg, ResType, I, true); 485 case TargetOpcode::G_ANYEXT: 486 case TargetOpcode::G_ZEXT: 487 return selectExt(ResVReg, ResType, I, false); 488 case TargetOpcode::G_TRUNC: 489 return selectTrunc(ResVReg, ResType, I); 490 case TargetOpcode::G_FPTRUNC: 491 case TargetOpcode::G_FPEXT: 492 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 493 494 case TargetOpcode::G_PTRTOINT: 495 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 496 case TargetOpcode::G_INTTOPTR: 497 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 498 case TargetOpcode::G_BITCAST: 499 return selectBitcast(ResVReg, ResType, I); 500 case TargetOpcode::G_ADDRSPACE_CAST: 501 return selectAddrSpaceCast(ResVReg, ResType, I); 502 case TargetOpcode::G_PTR_ADD: { 503 // Currently, we get G_PTR_ADD only as a result of translating 504 // global variables, initialized with constant expressions like GV + Const 505 // (see test opencl/basic/progvar_prog_scope_init.ll). 506 // TODO: extend the handler once we have other cases. 507 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 508 Register GV = I.getOperand(1).getReg(); 509 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 510 (void)II; 511 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 512 (*II).getOpcode() == TargetOpcode::COPY || 513 (*II).getOpcode() == SPIRV::OpVariable) && 514 isImm(I.getOperand(2), MRI)); 515 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 516 MachineBasicBlock &BB = *I.getParent(); 517 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 518 .addDef(ResVReg) 519 .addUse(GR.getSPIRVTypeID(ResType)) 520 .addImm(static_cast<uint32_t>( 521 SPIRV::Opcode::InBoundsPtrAccessChain)) 522 .addUse(GV) 523 .addUse(Idx) 524 .addUse(I.getOperand(2).getReg()); 525 return MIB.constrainAllUses(TII, TRI, RBI); 526 } 527 528 case TargetOpcode::G_ATOMICRMW_OR: 529 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 530 case TargetOpcode::G_ATOMICRMW_ADD: 531 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 532 case TargetOpcode::G_ATOMICRMW_AND: 533 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 534 case TargetOpcode::G_ATOMICRMW_MAX: 535 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 536 case TargetOpcode::G_ATOMICRMW_MIN: 537 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 538 case TargetOpcode::G_ATOMICRMW_SUB: 539 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 540 case TargetOpcode::G_ATOMICRMW_XOR: 541 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 542 case TargetOpcode::G_ATOMICRMW_UMAX: 543 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 544 case TargetOpcode::G_ATOMICRMW_UMIN: 545 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 546 case TargetOpcode::G_ATOMICRMW_XCHG: 547 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 548 case TargetOpcode::G_ATOMIC_CMPXCHG: 549 return selectAtomicCmpXchg(ResVReg, ResType, I); 550 551 case TargetOpcode::G_ATOMICRMW_FADD: 552 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 553 case TargetOpcode::G_ATOMICRMW_FSUB: 554 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 555 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 556 SPIRV::OpFNegate); 557 case TargetOpcode::G_ATOMICRMW_FMIN: 558 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 559 case TargetOpcode::G_ATOMICRMW_FMAX: 560 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 561 562 case TargetOpcode::G_FENCE: 563 return selectFence(I); 564 565 case TargetOpcode::G_STACKSAVE: 566 return selectStackSave(ResVReg, ResType, I); 567 case TargetOpcode::G_STACKRESTORE: 568 return selectStackRestore(I); 569 570 case TargetOpcode::G_UNMERGE_VALUES: 571 return selectUnmergeValues(I); 572 573 default: 574 return false; 575 } 576 } 577 578 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 579 const SPIRVType *ResType, 580 MachineInstr &I, 581 CL::OpenCLExtInst CLInst) const { 582 return selectExtInst(ResVReg, ResType, I, 583 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 584 } 585 586 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 587 const SPIRVType *ResType, 588 MachineInstr &I, 589 CL::OpenCLExtInst CLInst, 590 GL::GLSLExtInst GLInst) const { 591 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 592 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 593 return selectExtInst(ResVReg, ResType, I, ExtInsts); 594 } 595 596 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 597 const SPIRVType *ResType, 598 MachineInstr &I, 599 const ExtInstList &Insts) const { 600 601 for (const auto &Ex : Insts) { 602 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 603 uint32_t Opcode = Ex.second; 604 if (STI.canUseExtInstSet(Set)) { 605 MachineBasicBlock &BB = *I.getParent(); 606 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 607 .addDef(ResVReg) 608 .addUse(GR.getSPIRVTypeID(ResType)) 609 .addImm(static_cast<uint32_t>(Set)) 610 .addImm(Opcode); 611 const unsigned NumOps = I.getNumOperands(); 612 for (unsigned i = 1; i < NumOps; ++i) 613 MIB.add(I.getOperand(i)); 614 return MIB.constrainAllUses(TII, TRI, RBI); 615 } 616 } 617 return false; 618 } 619 620 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 621 const SPIRVType *ResType, 622 MachineInstr &I, 623 Register SrcReg, 624 unsigned Opcode) const { 625 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 626 .addDef(ResVReg) 627 .addUse(GR.getSPIRVTypeID(ResType)) 628 .addUse(SrcReg) 629 .constrainAllUses(TII, TRI, RBI); 630 } 631 632 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 633 const SPIRVType *ResType, 634 MachineInstr &I, 635 unsigned Opcode) const { 636 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 637 Opcode); 638 } 639 640 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 641 const SPIRVType *ResType, 642 MachineInstr &I) const { 643 Register OpReg = I.getOperand(1).getReg(); 644 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 645 if (!GR.isBitcastCompatible(ResType, OpType)) 646 report_fatal_error("incompatible result and operand types in a bitcast"); 647 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 648 } 649 650 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord, 651 SPIRVMachineModuleInfo *MMI) { 652 if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID) 653 return SPIRV::Scope::Invocation; 654 else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID) 655 return SPIRV::Scope::Device; 656 else if (Ord == MMI->WorkGroupSSID) 657 return SPIRV::Scope::Workgroup; 658 else if (Ord == MMI->AllSVMDevicesSSID) 659 return SPIRV::Scope::CrossDevice; 660 else if (Ord == MMI->SubGroupSSID) 661 return SPIRV::Scope::Subgroup; 662 else 663 // OpenCL approach is: "The functions that do not have memory_scope argument 664 // have the same semantics as the corresponding functions with the 665 // memory_scope argument set to memory_scope_device." See ref.: // 666 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions 667 // In our case if the scope is unknown, assuming that SPIR-V code is to be 668 // consumed in an OpenCL environment, we use the same approach and set the 669 // scope to memory_scope_device. 670 return SPIRV::Scope::Device; 671 } 672 673 static void addMemoryOperands(MachineMemOperand *MemOp, 674 MachineInstrBuilder &MIB) { 675 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 676 if (MemOp->isVolatile()) 677 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 678 if (MemOp->isNonTemporal()) 679 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 680 if (MemOp->getAlign().value()) 681 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 682 683 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 684 MIB.addImm(SpvMemOp); 685 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 686 MIB.addImm(MemOp->getAlign().value()); 687 } 688 } 689 690 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 691 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 692 if (Flags & MachineMemOperand::Flags::MOVolatile) 693 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 694 if (Flags & MachineMemOperand::Flags::MONonTemporal) 695 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 696 697 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 698 MIB.addImm(SpvMemOp); 699 } 700 701 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 702 const SPIRVType *ResType, 703 MachineInstr &I) const { 704 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 705 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 706 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 707 .addDef(ResVReg) 708 .addUse(GR.getSPIRVTypeID(ResType)) 709 .addUse(Ptr); 710 if (!I.getNumMemOperands()) { 711 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 712 I.getOpcode() == 713 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 714 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 715 } else { 716 addMemoryOperands(*I.memoperands_begin(), MIB); 717 } 718 return MIB.constrainAllUses(TII, TRI, RBI); 719 } 720 721 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 722 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 723 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 724 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 725 MachineBasicBlock &BB = *I.getParent(); 726 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 727 .addUse(Ptr) 728 .addUse(StoreVal); 729 if (!I.getNumMemOperands()) { 730 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 731 I.getOpcode() == 732 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 733 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 734 } else { 735 addMemoryOperands(*I.memoperands_begin(), MIB); 736 } 737 return MIB.constrainAllUses(TII, TRI, RBI); 738 } 739 740 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 741 const SPIRVType *ResType, 742 MachineInstr &I) const { 743 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 744 report_fatal_error( 745 "llvm.stacksave intrinsic: this instruction requires the following " 746 "SPIR-V extension: SPV_INTEL_variable_length_array", 747 false); 748 MachineBasicBlock &BB = *I.getParent(); 749 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 750 .addDef(ResVReg) 751 .addUse(GR.getSPIRVTypeID(ResType)) 752 .constrainAllUses(TII, TRI, RBI); 753 } 754 755 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 756 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 757 report_fatal_error( 758 "llvm.stackrestore intrinsic: this instruction requires the following " 759 "SPIR-V extension: SPV_INTEL_variable_length_array", 760 false); 761 if (!I.getOperand(0).isReg()) 762 return false; 763 MachineBasicBlock &BB = *I.getParent(); 764 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 765 .addUse(I.getOperand(0).getReg()) 766 .constrainAllUses(TII, TRI, RBI); 767 } 768 769 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 770 MachineInstr &I) const { 771 MachineBasicBlock &BB = *I.getParent(); 772 Register SrcReg = I.getOperand(1).getReg(); 773 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 774 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 775 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 776 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 777 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 778 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 779 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); 780 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 781 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 782 // TODO: check if we have such GV, add init, use buildGlobalVariable. 783 Function &CurFunction = GR.CurMF->getFunction(); 784 Type *LLVMArrTy = 785 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 786 // Module takes ownership of the global var. 787 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 788 true, GlobalValue::InternalLinkage, 789 Constant::getNullValue(LLVMArrTy)); 790 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 791 GR.add(GV, GR.CurMF, VarReg); 792 793 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 794 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 795 .addDef(VarReg) 796 .addUse(GR.getSPIRVTypeID(VarTy)) 797 .addImm(SPIRV::StorageClass::UniformConstant) 798 .addUse(Const) 799 .constrainAllUses(TII, TRI, RBI); 800 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 801 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 802 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 803 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 804 } 805 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 806 .addUse(I.getOperand(0).getReg()) 807 .addUse(SrcReg) 808 .addUse(I.getOperand(2).getReg()); 809 if (I.getNumMemOperands()) 810 addMemoryOperands(*I.memoperands_begin(), MIB); 811 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 812 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 813 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 814 .addUse(MIB->getOperand(0).getReg()); 815 return Result; 816 } 817 818 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 819 const SPIRVType *ResType, 820 MachineInstr &I, 821 unsigned NewOpcode, 822 unsigned NegateOpcode) const { 823 assert(I.hasOneMemOperand()); 824 const MachineMemOperand *MemOp = *I.memoperands_begin(); 825 uint32_t Scope = 826 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); 827 Register ScopeReg = buildI32Constant(Scope, I); 828 829 Register Ptr = I.getOperand(1).getReg(); 830 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 831 // auto ScSem = 832 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 833 AtomicOrdering AO = MemOp->getSuccessOrdering(); 834 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 835 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 836 837 bool Result = false; 838 Register ValueReg = I.getOperand(2).getReg(); 839 if (NegateOpcode != 0) { 840 // Translation with negative value operand is requested 841 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 842 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 843 ValueReg = TmpReg; 844 } 845 846 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 847 .addDef(ResVReg) 848 .addUse(GR.getSPIRVTypeID(ResType)) 849 .addUse(Ptr) 850 .addUse(ScopeReg) 851 .addUse(MemSemReg) 852 .addUse(ValueReg) 853 .constrainAllUses(TII, TRI, RBI); 854 return Result; 855 } 856 857 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 858 unsigned ArgI = I.getNumOperands() - 1; 859 Register SrcReg = 860 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 861 SPIRVType *DefType = 862 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 863 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 864 report_fatal_error( 865 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 866 867 SPIRVType *ScalarType = 868 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 869 MachineBasicBlock &BB = *I.getParent(); 870 bool Res = false; 871 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 872 Register ResVReg = I.getOperand(i).getReg(); 873 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 874 if (!ResType) { 875 // There was no "assign type" actions, let's fix this now 876 ResType = ScalarType; 877 MRI->setRegClass(ResVReg, &SPIRV::IDRegClass); 878 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 879 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 880 } 881 auto MIB = 882 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 883 .addDef(ResVReg) 884 .addUse(GR.getSPIRVTypeID(ResType)) 885 .addUse(SrcReg) 886 .addImm(static_cast<int64_t>(i)); 887 Res |= MIB.constrainAllUses(TII, TRI, RBI); 888 } 889 return Res; 890 } 891 892 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 893 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 894 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 895 Register MemSemReg = buildI32Constant(MemSem, I); 896 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 897 uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI)); 898 Register ScopeReg = buildI32Constant(Scope, I); 899 MachineBasicBlock &BB = *I.getParent(); 900 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 901 .addUse(ScopeReg) 902 .addUse(MemSemReg) 903 .constrainAllUses(TII, TRI, RBI); 904 } 905 906 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 907 const SPIRVType *ResType, 908 MachineInstr &I) const { 909 Register ScopeReg; 910 Register MemSemEqReg; 911 Register MemSemNeqReg; 912 Register Ptr = I.getOperand(2).getReg(); 913 if (!isa<GIntrinsic>(I)) { 914 assert(I.hasOneMemOperand()); 915 const MachineMemOperand *MemOp = *I.memoperands_begin(); 916 unsigned Scope = 917 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); 918 ScopeReg = buildI32Constant(Scope, I); 919 920 unsigned ScSem = static_cast<uint32_t>( 921 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 922 AtomicOrdering AO = MemOp->getSuccessOrdering(); 923 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 924 MemSemEqReg = buildI32Constant(MemSemEq, I); 925 AtomicOrdering FO = MemOp->getFailureOrdering(); 926 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 927 MemSemNeqReg = 928 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 929 } else { 930 ScopeReg = I.getOperand(5).getReg(); 931 MemSemEqReg = I.getOperand(6).getReg(); 932 MemSemNeqReg = I.getOperand(7).getReg(); 933 } 934 935 Register Cmp = I.getOperand(3).getReg(); 936 Register Val = I.getOperand(4).getReg(); 937 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 938 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); 939 const DebugLoc &DL = I.getDebugLoc(); 940 bool Result = 941 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 942 .addDef(ACmpRes) 943 .addUse(GR.getSPIRVTypeID(SpvValTy)) 944 .addUse(Ptr) 945 .addUse(ScopeReg) 946 .addUse(MemSemEqReg) 947 .addUse(MemSemNeqReg) 948 .addUse(Val) 949 .addUse(Cmp) 950 .constrainAllUses(TII, TRI, RBI); 951 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 952 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 953 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 954 .addDef(CmpSuccReg) 955 .addUse(GR.getSPIRVTypeID(BoolTy)) 956 .addUse(ACmpRes) 957 .addUse(Cmp) 958 .constrainAllUses(TII, TRI, RBI); 959 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 960 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 961 .addDef(TmpReg) 962 .addUse(GR.getSPIRVTypeID(ResType)) 963 .addUse(ACmpRes) 964 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 965 .addImm(0) 966 .constrainAllUses(TII, TRI, RBI); 967 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 968 .addDef(ResVReg) 969 .addUse(GR.getSPIRVTypeID(ResType)) 970 .addUse(CmpSuccReg) 971 .addUse(TmpReg) 972 .addImm(1) 973 .constrainAllUses(TII, TRI, RBI); 974 return Result; 975 } 976 977 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 978 switch (SC) { 979 case SPIRV::StorageClass::Workgroup: 980 case SPIRV::StorageClass::CrossWorkgroup: 981 case SPIRV::StorageClass::Function: 982 return true; 983 default: 984 return false; 985 } 986 } 987 988 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 989 switch (SC) { 990 case SPIRV::StorageClass::DeviceOnlyINTEL: 991 case SPIRV::StorageClass::HostOnlyINTEL: 992 return true; 993 default: 994 return false; 995 } 996 } 997 998 // In SPIR-V address space casting can only happen to and from the Generic 999 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1000 // pointers to and from Generic pointers. As such, we can convert e.g. from 1001 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1002 // other combinations can only be done by a bitcast, and are probably not safe. 1003 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1004 const SPIRVType *ResType, 1005 MachineInstr &I) const { 1006 // If the AddrSpaceCast user is single and in OpConstantComposite or 1007 // OpVariable, we should select OpSpecConstantOp. 1008 auto UIs = MRI->use_instructions(ResVReg); 1009 if (!UIs.empty() && ++UIs.begin() == UIs.end() && 1010 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || 1011 UIs.begin()->getOpcode() == SPIRV::OpVariable || 1012 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { 1013 Register NewReg = I.getOperand(1).getReg(); 1014 MachineBasicBlock &BB = *I.getParent(); 1015 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1016 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, 1017 SPIRV::StorageClass::Generic); 1018 bool Result = 1019 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 1020 .addDef(ResVReg) 1021 .addUse(GR.getSPIRVTypeID(ResType)) 1022 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) 1023 .addUse(NewReg) 1024 .constrainAllUses(TII, TRI, RBI); 1025 return Result; 1026 } 1027 Register SrcPtr = I.getOperand(1).getReg(); 1028 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1029 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); 1030 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); 1031 1032 // don't generate a cast between identical storage classes 1033 if (SrcSC == DstSC) 1034 return true; 1035 1036 // Casting from an eligible pointer to Generic. 1037 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1038 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1039 // Casting from Generic to an eligible pointer. 1040 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1041 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1042 // Casting between 2 eligible pointers using Generic as an intermediary. 1043 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1044 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1045 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1046 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); 1047 MachineBasicBlock &BB = *I.getParent(); 1048 const DebugLoc &DL = I.getDebugLoc(); 1049 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1050 .addDef(Tmp) 1051 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1052 .addUse(SrcPtr) 1053 .constrainAllUses(TII, TRI, RBI); 1054 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1055 .addDef(ResVReg) 1056 .addUse(GR.getSPIRVTypeID(ResType)) 1057 .addUse(Tmp) 1058 .constrainAllUses(TII, TRI, RBI); 1059 } 1060 1061 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1062 // be applied 1063 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1064 return selectUnOp(ResVReg, ResType, I, 1065 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1066 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1067 return selectUnOp(ResVReg, ResType, I, 1068 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1069 1070 // TODO Should this case just be disallowed completely? 1071 // We're casting 2 other arbitrary address spaces, so have to bitcast. 1072 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1073 } 1074 1075 static unsigned getFCmpOpcode(unsigned PredNum) { 1076 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1077 switch (Pred) { 1078 case CmpInst::FCMP_OEQ: 1079 return SPIRV::OpFOrdEqual; 1080 case CmpInst::FCMP_OGE: 1081 return SPIRV::OpFOrdGreaterThanEqual; 1082 case CmpInst::FCMP_OGT: 1083 return SPIRV::OpFOrdGreaterThan; 1084 case CmpInst::FCMP_OLE: 1085 return SPIRV::OpFOrdLessThanEqual; 1086 case CmpInst::FCMP_OLT: 1087 return SPIRV::OpFOrdLessThan; 1088 case CmpInst::FCMP_ONE: 1089 return SPIRV::OpFOrdNotEqual; 1090 case CmpInst::FCMP_ORD: 1091 return SPIRV::OpOrdered; 1092 case CmpInst::FCMP_UEQ: 1093 return SPIRV::OpFUnordEqual; 1094 case CmpInst::FCMP_UGE: 1095 return SPIRV::OpFUnordGreaterThanEqual; 1096 case CmpInst::FCMP_UGT: 1097 return SPIRV::OpFUnordGreaterThan; 1098 case CmpInst::FCMP_ULE: 1099 return SPIRV::OpFUnordLessThanEqual; 1100 case CmpInst::FCMP_ULT: 1101 return SPIRV::OpFUnordLessThan; 1102 case CmpInst::FCMP_UNE: 1103 return SPIRV::OpFUnordNotEqual; 1104 case CmpInst::FCMP_UNO: 1105 return SPIRV::OpUnordered; 1106 default: 1107 llvm_unreachable("Unknown predicate type for FCmp"); 1108 } 1109 } 1110 1111 static unsigned getICmpOpcode(unsigned PredNum) { 1112 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1113 switch (Pred) { 1114 case CmpInst::ICMP_EQ: 1115 return SPIRV::OpIEqual; 1116 case CmpInst::ICMP_NE: 1117 return SPIRV::OpINotEqual; 1118 case CmpInst::ICMP_SGE: 1119 return SPIRV::OpSGreaterThanEqual; 1120 case CmpInst::ICMP_SGT: 1121 return SPIRV::OpSGreaterThan; 1122 case CmpInst::ICMP_SLE: 1123 return SPIRV::OpSLessThanEqual; 1124 case CmpInst::ICMP_SLT: 1125 return SPIRV::OpSLessThan; 1126 case CmpInst::ICMP_UGE: 1127 return SPIRV::OpUGreaterThanEqual; 1128 case CmpInst::ICMP_UGT: 1129 return SPIRV::OpUGreaterThan; 1130 case CmpInst::ICMP_ULE: 1131 return SPIRV::OpULessThanEqual; 1132 case CmpInst::ICMP_ULT: 1133 return SPIRV::OpULessThan; 1134 default: 1135 llvm_unreachable("Unknown predicate type for ICmp"); 1136 } 1137 } 1138 1139 static unsigned getPtrCmpOpcode(unsigned Pred) { 1140 switch (static_cast<CmpInst::Predicate>(Pred)) { 1141 case CmpInst::ICMP_EQ: 1142 return SPIRV::OpPtrEqual; 1143 case CmpInst::ICMP_NE: 1144 return SPIRV::OpPtrNotEqual; 1145 default: 1146 llvm_unreachable("Unknown predicate type for pointer comparison"); 1147 } 1148 } 1149 1150 // Return the logical operation, or abort if none exists. 1151 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1152 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1153 switch (Pred) { 1154 case CmpInst::ICMP_EQ: 1155 return SPIRV::OpLogicalEqual; 1156 case CmpInst::ICMP_NE: 1157 return SPIRV::OpLogicalNotEqual; 1158 default: 1159 llvm_unreachable("Unknown predicate type for Bool comparison"); 1160 } 1161 } 1162 1163 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1164 const SPIRVType *ResType, 1165 MachineInstr &I) const { 1166 assert(I.getNumOperands() == 3); 1167 assert(I.getOperand(2).isReg()); 1168 MachineBasicBlock &BB = *I.getParent(); 1169 Register InputRegister = I.getOperand(2).getReg(); 1170 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1171 1172 if (!InputType) 1173 report_fatal_error("Input Type could not be determined."); 1174 1175 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1176 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1177 if (IsBoolTy && !IsVectorTy) { 1178 assert(ResVReg == I.getOperand(0).getReg()); 1179 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1180 TII.get(TargetOpcode::COPY)) 1181 .addDef(ResVReg) 1182 .addUse(InputRegister) 1183 .constrainAllUses(TII, TRI, RBI); 1184 } 1185 1186 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1187 unsigned SpirvNotEqualId = 1188 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1189 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1190 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1191 Register NotEqualReg = ResVReg; 1192 1193 if (IsVectorTy) { 1194 NotEqualReg = IsBoolTy ? InputRegister 1195 : MRI->createVirtualRegister(&SPIRV::IDRegClass); 1196 const unsigned NumElts = InputType->getOperand(2).getImm(); 1197 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1198 } 1199 1200 if (!IsBoolTy) { 1201 Register ConstZeroReg = 1202 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1203 1204 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1205 .addDef(NotEqualReg) 1206 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1207 .addUse(InputRegister) 1208 .addUse(ConstZeroReg) 1209 .constrainAllUses(TII, TRI, RBI); 1210 } 1211 1212 if (!IsVectorTy) 1213 return true; 1214 1215 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAll)) 1216 .addDef(ResVReg) 1217 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1218 .addUse(NotEqualReg) 1219 .constrainAllUses(TII, TRI, RBI); 1220 } 1221 1222 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1223 const SPIRVType *ResType, 1224 MachineInstr &I) const { 1225 MachineBasicBlock &BB = *I.getParent(); 1226 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1227 .addDef(ResVReg) 1228 .addUse(GR.getSPIRVTypeID(ResType)) 1229 .addUse(I.getOperand(1).getReg()) 1230 .constrainAllUses(TII, TRI, RBI); 1231 } 1232 1233 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1234 const SPIRVType *ResType, 1235 MachineInstr &I) const { 1236 // There is no way to implement `freeze` correctly without support on SPIR-V 1237 // standard side, but we may at least address a simple (static) case when 1238 // undef/poison value presence is obvious. The main benefit of even 1239 // incomplete `freeze` support is preventing of translation from crashing due 1240 // to lack of support on legalization and instruction selection steps. 1241 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1242 return false; 1243 Register OpReg = I.getOperand(1).getReg(); 1244 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1245 Register Reg; 1246 switch (Def->getOpcode()) { 1247 case SPIRV::ASSIGN_TYPE: 1248 if (MachineInstr *AssignToDef = 1249 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1250 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1251 Reg = Def->getOperand(2).getReg(); 1252 } 1253 break; 1254 case SPIRV::OpUndef: 1255 Reg = Def->getOperand(1).getReg(); 1256 break; 1257 } 1258 unsigned DestOpCode; 1259 if (Reg.isValid()) { 1260 DestOpCode = SPIRV::OpConstantNull; 1261 } else { 1262 DestOpCode = TargetOpcode::COPY; 1263 Reg = OpReg; 1264 } 1265 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1266 .addDef(I.getOperand(0).getReg()) 1267 .addUse(Reg) 1268 .constrainAllUses(TII, TRI, RBI); 1269 } 1270 return false; 1271 } 1272 1273 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, 1274 const SPIRVType *ResType, 1275 MachineInstr &I) const { 1276 // TODO: only const case is supported for now. 1277 assert(std::all_of( 1278 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { 1279 if (MO.isDef()) 1280 return true; 1281 if (!MO.isReg()) 1282 return false; 1283 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); 1284 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 1285 ConstTy->getOperand(1).isReg()); 1286 Register ConstReg = ConstTy->getOperand(1).getReg(); 1287 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 1288 assert(Const); 1289 return (Const->getOpcode() == TargetOpcode::G_CONSTANT || 1290 Const->getOpcode() == TargetOpcode::G_FCONSTANT); 1291 })); 1292 1293 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1294 TII.get(SPIRV::OpConstantComposite)) 1295 .addDef(ResVReg) 1296 .addUse(GR.getSPIRVTypeID(ResType)); 1297 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 1298 MIB.addUse(I.getOperand(i).getReg()); 1299 return MIB.constrainAllUses(TII, TRI, RBI); 1300 } 1301 1302 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1303 const SPIRVType *ResType) { 1304 Register OpReg = ResType->getOperand(2).getReg(); 1305 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1306 if (!OpDef) 1307 return 0; 1308 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1309 OpDef->getOperand(1).isReg()) { 1310 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1311 OpDef = RefDef; 1312 } 1313 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 1314 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 1315 : 0; 1316 return N; 1317 } 1318 1319 // Return true if the type represents a constant register 1320 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { 1321 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1322 OpDef->getOperand(1).isReg()) { 1323 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1324 OpDef = RefDef; 1325 } 1326 return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || 1327 OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; 1328 } 1329 1330 // Return true if the virtual register represents a constant 1331 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 1332 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 1333 return isConstReg(MRI, OpDef); 1334 return false; 1335 } 1336 1337 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 1338 const SPIRVType *ResType, 1339 MachineInstr &I) const { 1340 unsigned N = 0; 1341 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1342 N = GR.getScalarOrVectorComponentCount(ResType); 1343 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1344 N = getArrayComponentCount(MRI, ResType); 1345 else 1346 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 1347 1348 unsigned OpIdx = I.getNumExplicitDefs(); 1349 if (!I.getOperand(OpIdx).isReg()) 1350 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 1351 1352 // check if we may construct a constant vector 1353 Register OpReg = I.getOperand(OpIdx).getReg(); 1354 bool IsConst = isConstReg(MRI, OpReg); 1355 1356 if (!IsConst && N < 2) 1357 report_fatal_error( 1358 "There must be at least two constituent operands in a vector"); 1359 1360 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1361 TII.get(IsConst ? SPIRV::OpConstantComposite 1362 : SPIRV::OpCompositeConstruct)) 1363 .addDef(ResVReg) 1364 .addUse(GR.getSPIRVTypeID(ResType)); 1365 for (unsigned i = 0; i < N; ++i) 1366 MIB.addUse(OpReg); 1367 return MIB.constrainAllUses(TII, TRI, RBI); 1368 } 1369 1370 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 1371 const SPIRVType *ResType, 1372 unsigned CmpOpc, 1373 MachineInstr &I) const { 1374 Register Cmp0 = I.getOperand(2).getReg(); 1375 Register Cmp1 = I.getOperand(3).getReg(); 1376 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1377 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1378 "CMP operands should have the same type"); 1379 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1380 .addDef(ResVReg) 1381 .addUse(GR.getSPIRVTypeID(ResType)) 1382 .addUse(Cmp0) 1383 .addUse(Cmp1) 1384 .constrainAllUses(TII, TRI, RBI); 1385 } 1386 1387 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1388 const SPIRVType *ResType, 1389 MachineInstr &I) const { 1390 auto Pred = I.getOperand(1).getPredicate(); 1391 unsigned CmpOpc; 1392 1393 Register CmpOperand = I.getOperand(2).getReg(); 1394 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1395 CmpOpc = getPtrCmpOpcode(Pred); 1396 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1397 CmpOpc = getBoolCmpOpcode(Pred); 1398 else 1399 CmpOpc = getICmpOpcode(Pred); 1400 return selectCmp(ResVReg, ResType, CmpOpc, I); 1401 } 1402 1403 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, 1404 const MachineInstr &I, 1405 int OpIdx) const { 1406 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1407 "Expected G_FCONSTANT"); 1408 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1409 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1410 } 1411 1412 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1413 const MachineInstr &I, 1414 int OpIdx) const { 1415 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1416 "Expected G_CONSTANT"); 1417 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1418 } 1419 1420 Register 1421 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1422 const SPIRVType *ResType) const { 1423 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1424 const SPIRVType *SpvI32Ty = 1425 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1426 // Find a constant in DT or build a new one. 1427 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1428 Register NewReg = GR.find(ConstInt, GR.CurMF); 1429 if (!NewReg.isValid()) { 1430 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1431 GR.add(ConstInt, GR.CurMF, NewReg); 1432 MachineInstr *MI; 1433 MachineBasicBlock &BB = *I.getParent(); 1434 if (Val == 0) { 1435 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1436 .addDef(NewReg) 1437 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 1438 } else { 1439 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1440 .addDef(NewReg) 1441 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 1442 .addImm(APInt(32, Val).getZExtValue()); 1443 } 1444 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 1445 } 1446 return NewReg; 1447 } 1448 1449 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 1450 const SPIRVType *ResType, 1451 MachineInstr &I) const { 1452 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 1453 return selectCmp(ResVReg, ResType, CmpOp, I); 1454 } 1455 1456 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 1457 MachineInstr &I) const { 1458 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 1459 bool ZeroAsNull = STI.isOpenCLEnv(); 1460 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1461 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 1462 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 1463 } 1464 1465 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1466 if (!LLVMFloatTy) 1467 return APFloat::getZero(APFloat::IEEEsingle()); 1468 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1469 case Type::HalfTyID: 1470 return APFloat::getZero(APFloat::IEEEhalf()); 1471 default: 1472 case Type::FloatTyID: 1473 return APFloat::getZero(APFloat::IEEEsingle()); 1474 case Type::DoubleTyID: 1475 return APFloat::getZero(APFloat::IEEEdouble()); 1476 } 1477 } 1478 1479 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 1480 MachineInstr &I) const { 1481 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 1482 bool ZeroAsNull = STI.isOpenCLEnv(); 1483 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 1484 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1485 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 1486 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 1487 } 1488 1489 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 1490 const SPIRVType *ResType, 1491 MachineInstr &I) const { 1492 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1493 APInt One = 1494 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 1495 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1496 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 1497 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 1498 } 1499 1500 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 1501 const SPIRVType *ResType, 1502 MachineInstr &I, 1503 bool IsSigned) const { 1504 // To extend a bool, we need to use OpSelect between constants. 1505 Register ZeroReg = buildZerosVal(ResType, I); 1506 Register OneReg = buildOnesVal(IsSigned, ResType, I); 1507 bool IsScalarBool = 1508 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 1509 unsigned Opcode = 1510 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 1511 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1512 .addDef(ResVReg) 1513 .addUse(GR.getSPIRVTypeID(ResType)) 1514 .addUse(I.getOperand(1).getReg()) 1515 .addUse(OneReg) 1516 .addUse(ZeroReg) 1517 .constrainAllUses(TII, TRI, RBI); 1518 } 1519 1520 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 1521 const SPIRVType *ResType, 1522 MachineInstr &I, bool IsSigned, 1523 unsigned Opcode) const { 1524 Register SrcReg = I.getOperand(1).getReg(); 1525 // We can convert bool value directly to float type without OpConvert*ToF, 1526 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 1527 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 1528 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1529 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 1530 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 1531 const unsigned NumElts = ResType->getOperand(2).getImm(); 1532 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 1533 } 1534 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1535 selectSelect(SrcReg, TmpType, I, false); 1536 } 1537 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 1538 } 1539 1540 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 1541 const SPIRVType *ResType, 1542 MachineInstr &I, bool IsSigned) const { 1543 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) 1544 return selectSelect(ResVReg, ResType, I, IsSigned); 1545 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1546 return selectUnOp(ResVReg, ResType, I, Opcode); 1547 } 1548 1549 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 1550 Register ResVReg, 1551 MachineInstr &I, 1552 const SPIRVType *IntTy, 1553 const SPIRVType *BoolTy) const { 1554 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 1555 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1556 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 1557 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 1558 Register Zero = buildZerosVal(IntTy, I); 1559 Register One = buildOnesVal(false, IntTy, I); 1560 MachineBasicBlock &BB = *I.getParent(); 1561 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1562 .addDef(BitIntReg) 1563 .addUse(GR.getSPIRVTypeID(IntTy)) 1564 .addUse(IntReg) 1565 .addUse(One) 1566 .constrainAllUses(TII, TRI, RBI); 1567 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1568 .addDef(ResVReg) 1569 .addUse(GR.getSPIRVTypeID(BoolTy)) 1570 .addUse(BitIntReg) 1571 .addUse(Zero) 1572 .constrainAllUses(TII, TRI, RBI); 1573 } 1574 1575 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 1576 const SPIRVType *ResType, 1577 MachineInstr &I) const { 1578 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { 1579 Register IntReg = I.getOperand(1).getReg(); 1580 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 1581 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 1582 } 1583 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 1584 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1585 return selectUnOp(ResVReg, ResType, I, Opcode); 1586 } 1587 1588 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 1589 const SPIRVType *ResType, 1590 const APInt &Imm, 1591 MachineInstr &I) const { 1592 unsigned TyOpcode = ResType->getOpcode(); 1593 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 1594 MachineBasicBlock &BB = *I.getParent(); 1595 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 1596 Imm.isZero()) 1597 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1598 .addDef(ResVReg) 1599 .addUse(GR.getSPIRVTypeID(ResType)) 1600 .constrainAllUses(TII, TRI, RBI); 1601 if (TyOpcode == SPIRV::OpTypeInt) { 1602 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 1603 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 1604 if (Reg == ResVReg) 1605 return true; 1606 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1607 .addDef(ResVReg) 1608 .addUse(Reg) 1609 .constrainAllUses(TII, TRI, RBI); 1610 } 1611 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1612 .addDef(ResVReg) 1613 .addUse(GR.getSPIRVTypeID(ResType)); 1614 // <=32-bit integers should be caught by the sdag pattern. 1615 assert(Imm.getBitWidth() > 32); 1616 addNumImm(Imm, MIB); 1617 return MIB.constrainAllUses(TII, TRI, RBI); 1618 } 1619 1620 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 1621 const SPIRVType *ResType, 1622 MachineInstr &I) const { 1623 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1624 .addDef(ResVReg) 1625 .addUse(GR.getSPIRVTypeID(ResType)) 1626 .constrainAllUses(TII, TRI, RBI); 1627 } 1628 1629 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1630 assert(MO.isReg()); 1631 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1632 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) 1633 return false; 1634 assert(TypeInst->getOperand(1).isReg()); 1635 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1636 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 1637 } 1638 1639 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1640 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1641 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1642 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 1643 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 1644 } 1645 1646 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 1647 const SPIRVType *ResType, 1648 MachineInstr &I) const { 1649 MachineBasicBlock &BB = *I.getParent(); 1650 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 1651 .addDef(ResVReg) 1652 .addUse(GR.getSPIRVTypeID(ResType)) 1653 // object to insert 1654 .addUse(I.getOperand(3).getReg()) 1655 // composite to insert into 1656 .addUse(I.getOperand(2).getReg()); 1657 for (unsigned i = 4; i < I.getNumOperands(); i++) 1658 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1659 return MIB.constrainAllUses(TII, TRI, RBI); 1660 } 1661 1662 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 1663 const SPIRVType *ResType, 1664 MachineInstr &I) const { 1665 MachineBasicBlock &BB = *I.getParent(); 1666 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1667 .addDef(ResVReg) 1668 .addUse(GR.getSPIRVTypeID(ResType)) 1669 .addUse(I.getOperand(2).getReg()); 1670 for (unsigned i = 3; i < I.getNumOperands(); i++) 1671 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1672 return MIB.constrainAllUses(TII, TRI, RBI); 1673 } 1674 1675 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 1676 const SPIRVType *ResType, 1677 MachineInstr &I) const { 1678 if (isImm(I.getOperand(4), MRI)) 1679 return selectInsertVal(ResVReg, ResType, I); 1680 MachineBasicBlock &BB = *I.getParent(); 1681 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 1682 .addDef(ResVReg) 1683 .addUse(GR.getSPIRVTypeID(ResType)) 1684 .addUse(I.getOperand(2).getReg()) 1685 .addUse(I.getOperand(3).getReg()) 1686 .addUse(I.getOperand(4).getReg()) 1687 .constrainAllUses(TII, TRI, RBI); 1688 } 1689 1690 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 1691 const SPIRVType *ResType, 1692 MachineInstr &I) const { 1693 if (isImm(I.getOperand(3), MRI)) 1694 return selectExtractVal(ResVReg, ResType, I); 1695 MachineBasicBlock &BB = *I.getParent(); 1696 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 1697 .addDef(ResVReg) 1698 .addUse(GR.getSPIRVTypeID(ResType)) 1699 .addUse(I.getOperand(2).getReg()) 1700 .addUse(I.getOperand(3).getReg()) 1701 .constrainAllUses(TII, TRI, RBI); 1702 } 1703 1704 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 1705 const SPIRVType *ResType, 1706 MachineInstr &I) const { 1707 const bool IsGEPInBounds = I.getOperand(2).getImm(); 1708 1709 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 1710 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 1711 // we have to use Op[InBounds]AccessChain. 1712 const unsigned Opcode = STI.isVulkanEnv() 1713 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 1714 : SPIRV::OpAccessChain) 1715 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 1716 : SPIRV::OpPtrAccessChain); 1717 1718 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1719 .addDef(ResVReg) 1720 .addUse(GR.getSPIRVTypeID(ResType)) 1721 // Object to get a pointer to. 1722 .addUse(I.getOperand(3).getReg()); 1723 // Adding indices. 1724 const unsigned StartingIndex = 1725 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 1726 ? 5 1727 : 4; 1728 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 1729 Res.addUse(I.getOperand(i).getReg()); 1730 return Res.constrainAllUses(TII, TRI, RBI); 1731 } 1732 1733 // Maybe wrap a value into OpSpecConstantOp 1734 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 1735 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 1736 bool Result = true; 1737 unsigned Lim = I.getNumExplicitOperands(); 1738 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 1739 Register OpReg = I.getOperand(i).getReg(); 1740 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 1741 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 1742 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || 1743 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { 1744 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 1745 // by selectAddrSpaceCast() 1746 CompositeArgs.push_back(OpReg); 1747 continue; 1748 } 1749 MachineFunction *MF = I.getMF(); 1750 Register WrapReg = GR.find(OpDefine, MF); 1751 if (WrapReg.isValid()) { 1752 CompositeArgs.push_back(WrapReg); 1753 continue; 1754 } 1755 // Create a new register for the wrapper 1756 WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1757 GR.add(OpDefine, MF, WrapReg); 1758 CompositeArgs.push_back(WrapReg); 1759 // Decorate the wrapper register and generate a new instruction 1760 MRI->setType(WrapReg, LLT::pointer(0, 32)); 1761 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 1762 MachineBasicBlock &BB = *I.getParent(); 1763 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 1764 .addDef(WrapReg) 1765 .addUse(GR.getSPIRVTypeID(OpType)) 1766 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 1767 .addUse(OpReg) 1768 .constrainAllUses(TII, TRI, RBI); 1769 if (!Result) 1770 break; 1771 } 1772 return Result; 1773 } 1774 1775 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 1776 const SPIRVType *ResType, 1777 MachineInstr &I) const { 1778 MachineBasicBlock &BB = *I.getParent(); 1779 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 1780 switch (IID) { 1781 case Intrinsic::spv_load: 1782 return selectLoad(ResVReg, ResType, I); 1783 case Intrinsic::spv_store: 1784 return selectStore(I); 1785 case Intrinsic::spv_extractv: 1786 return selectExtractVal(ResVReg, ResType, I); 1787 case Intrinsic::spv_insertv: 1788 return selectInsertVal(ResVReg, ResType, I); 1789 case Intrinsic::spv_extractelt: 1790 return selectExtractElt(ResVReg, ResType, I); 1791 case Intrinsic::spv_insertelt: 1792 return selectInsertElt(ResVReg, ResType, I); 1793 case Intrinsic::spv_gep: 1794 return selectGEP(ResVReg, ResType, I); 1795 case Intrinsic::spv_unref_global: 1796 case Intrinsic::spv_init_global: { 1797 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 1798 MachineInstr *Init = I.getNumExplicitOperands() > 2 1799 ? MRI->getVRegDef(I.getOperand(2).getReg()) 1800 : nullptr; 1801 assert(MI); 1802 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 1803 } 1804 case Intrinsic::spv_undef: { 1805 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1806 .addDef(ResVReg) 1807 .addUse(GR.getSPIRVTypeID(ResType)); 1808 return MIB.constrainAllUses(TII, TRI, RBI); 1809 } 1810 case Intrinsic::spv_const_composite: { 1811 // If no values are attached, the composite is null constant. 1812 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 1813 // Select a proper instruction. 1814 unsigned Opcode = SPIRV::OpConstantNull; 1815 SmallVector<Register> CompositeArgs; 1816 if (!IsNull) { 1817 Opcode = SPIRV::OpConstantComposite; 1818 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 1819 return false; 1820 } 1821 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1822 .addDef(ResVReg) 1823 .addUse(GR.getSPIRVTypeID(ResType)); 1824 // skip type MD node we already used when generated assign.type for this 1825 if (!IsNull) { 1826 for (Register OpReg : CompositeArgs) 1827 MIB.addUse(OpReg); 1828 } 1829 return MIB.constrainAllUses(TII, TRI, RBI); 1830 } 1831 case Intrinsic::spv_assign_name: { 1832 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 1833 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 1834 for (unsigned i = I.getNumExplicitDefs() + 2; 1835 i < I.getNumExplicitOperands(); ++i) { 1836 MIB.addImm(I.getOperand(i).getImm()); 1837 } 1838 return MIB.constrainAllUses(TII, TRI, RBI); 1839 } 1840 case Intrinsic::spv_switch: { 1841 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 1842 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 1843 if (I.getOperand(i).isReg()) 1844 MIB.addReg(I.getOperand(i).getReg()); 1845 else if (I.getOperand(i).isCImm()) 1846 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 1847 else if (I.getOperand(i).isMBB()) 1848 MIB.addMBB(I.getOperand(i).getMBB()); 1849 else 1850 llvm_unreachable("Unexpected OpSwitch operand"); 1851 } 1852 return MIB.constrainAllUses(TII, TRI, RBI); 1853 } 1854 case Intrinsic::spv_cmpxchg: 1855 return selectAtomicCmpXchg(ResVReg, ResType, I); 1856 case Intrinsic::spv_unreachable: 1857 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 1858 break; 1859 case Intrinsic::spv_alloca: 1860 return selectFrameIndex(ResVReg, ResType, I); 1861 case Intrinsic::spv_alloca_array: 1862 return selectAllocaArray(ResVReg, ResType, I); 1863 case Intrinsic::spv_assume: 1864 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1865 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 1866 .addUse(I.getOperand(1).getReg()); 1867 break; 1868 case Intrinsic::spv_expect: 1869 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1870 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 1871 .addDef(ResVReg) 1872 .addUse(GR.getSPIRVTypeID(ResType)) 1873 .addUse(I.getOperand(2).getReg()) 1874 .addUse(I.getOperand(3).getReg()); 1875 break; 1876 case Intrinsic::spv_thread_id: 1877 return selectSpvThreadId(ResVReg, ResType, I); 1878 case Intrinsic::spv_all: 1879 return selectAll(ResVReg, ResType, I); 1880 case Intrinsic::spv_lifetime_start: 1881 case Intrinsic::spv_lifetime_end: { 1882 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 1883 : SPIRV::OpLifetimeStop; 1884 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 1885 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 1886 unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg); 1887 bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid; 1888 if (Size == -1 || IsNonvoidPtr) 1889 Size = 0; 1890 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 1891 } break; 1892 default: { 1893 std::string DiagMsg; 1894 raw_string_ostream OS(DiagMsg); 1895 I.print(OS); 1896 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 1897 report_fatal_error(DiagMsg.c_str(), false); 1898 } 1899 } 1900 return true; 1901 } 1902 1903 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 1904 const SPIRVType *ResType, 1905 MachineInstr &I) const { 1906 // there was an allocation size parameter to the allocation instruction 1907 // that is not 1 1908 MachineBasicBlock &BB = *I.getParent(); 1909 return BuildMI(BB, I, I.getDebugLoc(), 1910 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 1911 .addDef(ResVReg) 1912 .addUse(GR.getSPIRVTypeID(ResType)) 1913 .addUse(I.getOperand(2).getReg()) 1914 .constrainAllUses(TII, TRI, RBI); 1915 } 1916 1917 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 1918 const SPIRVType *ResType, 1919 MachineInstr &I) const { 1920 // Change order of instructions if needed: all OpVariable instructions in a 1921 // function must be the first instructions in the first block 1922 MachineFunction *MF = I.getParent()->getParent(); 1923 MachineBasicBlock *MBB = &MF->front(); 1924 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 1925 bool IsHeader = false; 1926 unsigned Opcode; 1927 for (; It != E && It != I; ++It) { 1928 Opcode = It->getOpcode(); 1929 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 1930 IsHeader = true; 1931 } else if (IsHeader && 1932 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 1933 ++It; 1934 break; 1935 } 1936 } 1937 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 1938 .addDef(ResVReg) 1939 .addUse(GR.getSPIRVTypeID(ResType)) 1940 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 1941 .constrainAllUses(TII, TRI, RBI); 1942 } 1943 1944 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 1945 // InstructionSelector walks backwards through the instructions. We can use 1946 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 1947 // first, so can generate an OpBranchConditional here. If there is no 1948 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 1949 const MachineInstr *PrevI = I.getPrevNode(); 1950 MachineBasicBlock &MBB = *I.getParent(); 1951 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 1952 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1953 .addUse(PrevI->getOperand(0).getReg()) 1954 .addMBB(PrevI->getOperand(1).getMBB()) 1955 .addMBB(I.getOperand(0).getMBB()) 1956 .constrainAllUses(TII, TRI, RBI); 1957 } 1958 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 1959 .addMBB(I.getOperand(0).getMBB()) 1960 .constrainAllUses(TII, TRI, RBI); 1961 } 1962 1963 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 1964 // InstructionSelector walks backwards through the instructions. For an 1965 // explicit conditional branch with no fallthrough, we use both a G_BR and a 1966 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 1967 // generate the OpBranchConditional in selectBranch above. 1968 // 1969 // If an OpBranchConditional has been generated, we simply return, as the work 1970 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 1971 // implicit fallthrough to the next basic block, so we need to create an 1972 // OpBranchConditional with an explicit "false" argument pointing to the next 1973 // basic block that LLVM would fall through to. 1974 const MachineInstr *NextI = I.getNextNode(); 1975 // Check if this has already been successfully selected. 1976 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 1977 return true; 1978 // Must be relying on implicit block fallthrough, so generate an 1979 // OpBranchConditional with the "next" basic block as the "false" target. 1980 MachineBasicBlock &MBB = *I.getParent(); 1981 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 1982 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 1983 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1984 .addUse(I.getOperand(0).getReg()) 1985 .addMBB(I.getOperand(1).getMBB()) 1986 .addMBB(NextMBB) 1987 .constrainAllUses(TII, TRI, RBI); 1988 } 1989 1990 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 1991 const SPIRVType *ResType, 1992 MachineInstr &I) const { 1993 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 1994 .addDef(ResVReg) 1995 .addUse(GR.getSPIRVTypeID(ResType)); 1996 const unsigned NumOps = I.getNumOperands(); 1997 for (unsigned i = 1; i < NumOps; i += 2) { 1998 MIB.addUse(I.getOperand(i + 0).getReg()); 1999 MIB.addMBB(I.getOperand(i + 1).getMBB()); 2000 } 2001 return MIB.constrainAllUses(TII, TRI, RBI); 2002 } 2003 2004 bool SPIRVInstructionSelector::selectGlobalValue( 2005 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 2006 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 2007 MachineIRBuilder MIRBuilder(I); 2008 const GlobalValue *GV = I.getOperand(1).getGlobal(); 2009 Type *GVType = GR.getDeducedGlobalValueType(GV); 2010 SPIRVType *PointerBaseType; 2011 if (GVType->isArrayTy()) { 2012 SPIRVType *ArrayElementType = 2013 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 2014 SPIRV::AccessQualifier::ReadWrite, false); 2015 PointerBaseType = GR.getOrCreateSPIRVArrayType( 2016 ArrayElementType, GVType->getArrayNumElements(), I, TII); 2017 } else { 2018 PointerBaseType = GR.getOrCreateSPIRVType( 2019 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 2020 } 2021 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 2022 PointerBaseType, I, TII, 2023 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 2024 2025 std::string GlobalIdent; 2026 if (!GV->hasName()) { 2027 unsigned &ID = UnnamedGlobalIDs[GV]; 2028 if (ID == 0) 2029 ID = UnnamedGlobalIDs.size(); 2030 GlobalIdent = "__unnamed_" + Twine(ID).str(); 2031 } else { 2032 GlobalIdent = GV->getGlobalIdentifier(); 2033 } 2034 2035 // Behaviour of functions as operands depends on availability of the 2036 // corresponding extension (SPV_INTEL_function_pointers): 2037 // - If there is an extension to operate with functions as operands: 2038 // We create a proper constant operand and evaluate a correct type for a 2039 // function pointer. 2040 // - Without the required extension: 2041 // We have functions as operands in tests with blocks of instruction e.g. in 2042 // transcoding/global_block.ll. These operands are not used and should be 2043 // substituted by zero constants. Their type is expected to be always 2044 // OpTypePointer Function %uchar. 2045 if (isa<Function>(GV)) { 2046 const Constant *ConstVal = GV; 2047 MachineBasicBlock &BB = *I.getParent(); 2048 Register NewReg = GR.find(ConstVal, GR.CurMF); 2049 if (!NewReg.isValid()) { 2050 Register NewReg = ResVReg; 2051 GR.add(ConstVal, GR.CurMF, NewReg); 2052 const Function *GVFun = 2053 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 2054 ? dyn_cast<Function>(GV) 2055 : nullptr; 2056 if (GVFun) { 2057 // References to a function via function pointers generate virtual 2058 // registers without a definition. We will resolve it later, during 2059 // module analysis stage. 2060 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2061 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 2062 MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); 2063 MachineInstrBuilder MB = 2064 BuildMI(BB, I, I.getDebugLoc(), 2065 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 2066 .addDef(NewReg) 2067 .addUse(GR.getSPIRVTypeID(ResType)) 2068 .addUse(FuncVReg); 2069 // mapping the function pointer to the used Function 2070 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 2071 return MB.constrainAllUses(TII, TRI, RBI); 2072 } 2073 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2074 .addDef(NewReg) 2075 .addUse(GR.getSPIRVTypeID(ResType)) 2076 .constrainAllUses(TII, TRI, RBI); 2077 } 2078 assert(NewReg != ResVReg); 2079 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2080 .addDef(ResVReg) 2081 .addUse(NewReg) 2082 .constrainAllUses(TII, TRI, RBI); 2083 } 2084 auto GlobalVar = cast<GlobalVariable>(GV); 2085 assert(GlobalVar->getName() != "llvm.global.annotations"); 2086 2087 bool HasInit = GlobalVar->hasInitializer() && 2088 !isa<UndefValue>(GlobalVar->getInitializer()); 2089 // Skip empty declaration for GVs with initilaizers till we get the decl with 2090 // passed initializer. 2091 if (HasInit && !Init) 2092 return true; 2093 2094 unsigned AddrSpace = GV->getAddressSpace(); 2095 SPIRV::StorageClass::StorageClass Storage = 2096 addressSpaceToStorageClass(AddrSpace, STI); 2097 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 2098 Storage != SPIRV::StorageClass::Function; 2099 SPIRV::LinkageType::LinkageType LnkType = 2100 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 2101 ? SPIRV::LinkageType::Import 2102 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 2103 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 2104 ? SPIRV::LinkageType::LinkOnceODR 2105 : SPIRV::LinkageType::Export); 2106 2107 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 2108 Storage, Init, GlobalVar->isConstant(), 2109 HasLnkTy, LnkType, MIRBuilder, true); 2110 return Reg.isValid(); 2111 } 2112 2113 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 2114 const SPIRVType *ResType, 2115 MachineInstr &I) const { 2116 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 2117 return selectExtInst(ResVReg, ResType, I, CL::log10); 2118 } 2119 2120 // There is no log10 instruction in the GLSL Extended Instruction set, so it 2121 // is implemented as: 2122 // log10(x) = log2(x) * (1 / log2(10)) 2123 // = log2(x) * 0.30103 2124 2125 MachineIRBuilder MIRBuilder(I); 2126 MachineBasicBlock &BB = *I.getParent(); 2127 2128 // Build log2(x). 2129 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2130 bool Result = 2131 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2132 .addDef(VarReg) 2133 .addUse(GR.getSPIRVTypeID(ResType)) 2134 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2135 .addImm(GL::Log2) 2136 .add(I.getOperand(1)) 2137 .constrainAllUses(TII, TRI, RBI); 2138 2139 // Build 0.30103. 2140 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 2141 ResType->getOpcode() == SPIRV::OpTypeFloat); 2142 // TODO: Add matrix implementation once supported by the HLSL frontend. 2143 const SPIRVType *SpirvScalarType = 2144 ResType->getOpcode() == SPIRV::OpTypeVector 2145 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 2146 : ResType; 2147 Register ScaleReg = 2148 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 2149 2150 // Multiply log2(x) by 0.30103 to get log10(x) result. 2151 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 2152 ? SPIRV::OpVectorTimesScalar 2153 : SPIRV::OpFMulS; 2154 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2155 .addDef(ResVReg) 2156 .addUse(GR.getSPIRVTypeID(ResType)) 2157 .addUse(VarReg) 2158 .addUse(ScaleReg) 2159 .constrainAllUses(TII, TRI, RBI); 2160 2161 return Result; 2162 } 2163 2164 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 2165 const SPIRVType *ResType, 2166 MachineInstr &I) const { 2167 // DX intrinsic: @llvm.dx.thread.id(i32) 2168 // ID Name Description 2169 // 93 ThreadId reads the thread ID 2170 2171 MachineIRBuilder MIRBuilder(I); 2172 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 2173 const SPIRVType *Vec3Ty = 2174 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 2175 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 2176 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 2177 2178 // Create new register for GlobalInvocationID builtin variable. 2179 Register NewRegister = 2180 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); 2181 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32)); 2182 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 2183 2184 // Build GlobalInvocationID global variable with the necessary decorations. 2185 Register Variable = GR.buildGlobalVariable( 2186 NewRegister, PtrType, 2187 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 2188 SPIRV::StorageClass::Input, nullptr, true, true, 2189 SPIRV::LinkageType::Import, MIRBuilder, false); 2190 2191 // Create new register for loading value. 2192 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2193 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2194 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32)); 2195 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 2196 2197 // Load v3uint value from the global variable. 2198 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2199 .addDef(LoadedRegister) 2200 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 2201 .addUse(Variable); 2202 2203 // Get Thread ID index. Expecting operand is a constant immediate value, 2204 // wrapped in a type assignment. 2205 assert(I.getOperand(2).isReg()); 2206 Register ThreadIdReg = I.getOperand(2).getReg(); 2207 SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg); 2208 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 2209 ConstTy->getOperand(1).isReg()); 2210 Register ConstReg = ConstTy->getOperand(1).getReg(); 2211 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 2212 assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); 2213 const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue(); 2214 const uint32_t ThreadId = Val.getZExtValue(); 2215 2216 // Extract the thread ID from the loaded vector value. 2217 MachineBasicBlock &BB = *I.getParent(); 2218 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2219 .addDef(ResVReg) 2220 .addUse(GR.getSPIRVTypeID(ResType)) 2221 .addUse(LoadedRegister) 2222 .addImm(ThreadId); 2223 return MIB.constrainAllUses(TII, TRI, RBI); 2224 } 2225 2226 namespace llvm { 2227 InstructionSelector * 2228 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 2229 const SPIRVSubtarget &Subtarget, 2230 const RegisterBankInfo &RBI) { 2231 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 2232 } 2233 } // namespace llvm 2234