1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/CodeGen/TargetOpcodes.h" 32 #include "llvm/IR/IntrinsicsSPIRV.h" 33 #include "llvm/Support/Debug.h" 34 35 namespace llvm { 36 37 class SPIRVMachineModuleInfo : public MachineModuleInfoImpl { 38 public: 39 SyncScope::ID Work_ItemSSID; 40 SyncScope::ID WorkGroupSSID; 41 SyncScope::ID DeviceSSID; 42 SyncScope::ID AllSVMDevicesSSID; 43 SyncScope::ID SubGroupSSID; 44 45 SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) { 46 LLVMContext &CTX = MMI.getModule()->getContext(); 47 Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item"); 48 WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); 49 DeviceSSID = CTX.getOrInsertSyncScopeID("device"); 50 AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices"); 51 SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group"); 52 } 53 }; 54 55 } // end namespace llvm 56 57 #define DEBUG_TYPE "spirv-isel" 58 59 using namespace llvm; 60 namespace CL = SPIRV::OpenCLExtInst; 61 namespace GL = SPIRV::GLSLExtInst; 62 63 using ExtInstList = 64 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 65 66 namespace { 67 68 #define GET_GLOBALISEL_PREDICATE_BITSET 69 #include "SPIRVGenGlobalISel.inc" 70 #undef GET_GLOBALISEL_PREDICATE_BITSET 71 72 class SPIRVInstructionSelector : public InstructionSelector { 73 const SPIRVSubtarget &STI; 74 const SPIRVInstrInfo &TII; 75 const SPIRVRegisterInfo &TRI; 76 const RegisterBankInfo &RBI; 77 SPIRVGlobalRegistry &GR; 78 MachineRegisterInfo *MRI; 79 SPIRVMachineModuleInfo *MMI = nullptr; 80 81 /// We need to keep track of the number we give to anonymous global values to 82 /// generate the same name every time when this is needed. 83 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 84 85 public: 86 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 87 const SPIRVSubtarget &ST, 88 const RegisterBankInfo &RBI); 89 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 90 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 91 BlockFrequencyInfo *BFI) override; 92 // Common selection code. Instruction-specific selection occurs in spvSelect. 93 bool select(MachineInstr &I) override; 94 static const char *getName() { return DEBUG_TYPE; } 95 96 #define GET_GLOBALISEL_PREDICATES_DECL 97 #include "SPIRVGenGlobalISel.inc" 98 #undef GET_GLOBALISEL_PREDICATES_DECL 99 100 #define GET_GLOBALISEL_TEMPORARIES_DECL 101 #include "SPIRVGenGlobalISel.inc" 102 #undef GET_GLOBALISEL_TEMPORARIES_DECL 103 104 private: 105 // tblgen-erated 'select' implementation, used as the initial selector for 106 // the patterns that don't require complex C++. 107 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 108 109 // All instruction-specific selection that didn't happen in "select()". 110 // Is basically a large Switch/Case delegating to all other select method. 111 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 112 MachineInstr &I) const; 113 114 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 115 const MachineInstr *Init = nullptr) const; 116 117 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 118 MachineInstr &I, Register SrcReg, 119 unsigned Opcode) const; 120 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 121 unsigned Opcode) const; 122 123 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 124 MachineInstr &I) const; 125 126 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 127 MachineInstr &I) const; 128 bool selectStore(MachineInstr &I) const; 129 130 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 131 MachineInstr &I) const; 132 bool selectStackRestore(MachineInstr &I) const; 133 134 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 135 136 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 137 MachineInstr &I, unsigned NewOpcode, 138 unsigned NegateOpcode = 0) const; 139 140 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 141 MachineInstr &I) const; 142 143 bool selectFence(MachineInstr &I) const; 144 145 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 148 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, 149 MachineInstr &I, unsigned OpType) const; 150 151 bool selectAll(Register ResVReg, const SPIRVType *ResType, 152 MachineInstr &I) const; 153 154 bool selectAny(Register ResVReg, const SPIRVType *ResType, 155 MachineInstr &I) const; 156 157 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 158 MachineInstr &I) const; 159 160 bool selectConstVector(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 163 MachineInstr &I) const; 164 165 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 166 unsigned comparisonOpcode, MachineInstr &I) const; 167 168 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 171 MachineInstr &I) const; 172 173 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 174 int OpIdx) const; 175 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 176 int OpIdx) const; 177 178 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 179 MachineInstr &I) const; 180 181 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 182 bool IsSigned) const; 183 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 184 bool IsSigned, unsigned Opcode) const; 185 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 186 bool IsSigned) const; 187 188 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 191 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 192 const SPIRVType *intTy, const SPIRVType *boolTy) const; 193 194 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 195 MachineInstr &I) const; 196 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 199 MachineInstr &I) const; 200 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 201 MachineInstr &I) const; 202 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 203 MachineInstr &I) const; 204 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 205 MachineInstr &I) const; 206 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 207 MachineInstr &I) const; 208 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 209 MachineInstr &I) const; 210 211 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 212 MachineInstr &I) const; 213 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 214 MachineInstr &I) const; 215 216 bool selectBranch(MachineInstr &I) const; 217 bool selectBranchCond(MachineInstr &I) const; 218 219 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 220 MachineInstr &I) const; 221 222 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 223 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 224 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 225 MachineInstr &I, CL::OpenCLExtInst CLInst, 226 GL::GLSLExtInst GLInst) const; 227 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 228 MachineInstr &I, const ExtInstList &ExtInsts) const; 229 230 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 231 MachineInstr &I) const; 232 233 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 234 MachineInstr &I) const; 235 236 bool selectUnmergeValues(MachineInstr &I) const; 237 238 Register buildI32Constant(uint32_t Val, MachineInstr &I, 239 const SPIRVType *ResType = nullptr) const; 240 241 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 242 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; 243 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 244 MachineInstr &I) const; 245 246 bool wrapIntoSpecConstantOp(MachineInstr &I, 247 SmallVector<Register> &CompositeArgs) const; 248 }; 249 250 } // end anonymous namespace 251 252 #define GET_GLOBALISEL_IMPL 253 #include "SPIRVGenGlobalISel.inc" 254 #undef GET_GLOBALISEL_IMPL 255 256 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 257 const SPIRVSubtarget &ST, 258 const RegisterBankInfo &RBI) 259 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 260 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 261 #define GET_GLOBALISEL_PREDICATES_INIT 262 #include "SPIRVGenGlobalISel.inc" 263 #undef GET_GLOBALISEL_PREDICATES_INIT 264 #define GET_GLOBALISEL_TEMPORARIES_INIT 265 #include "SPIRVGenGlobalISel.inc" 266 #undef GET_GLOBALISEL_TEMPORARIES_INIT 267 { 268 } 269 270 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 271 CodeGenCoverage *CoverageInfo, 272 ProfileSummaryInfo *PSI, 273 BlockFrequencyInfo *BFI) { 274 MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>(); 275 MRI = &MF.getRegInfo(); 276 GR.setCurrentFunc(MF); 277 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 278 } 279 280 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 281 282 // Defined in SPIRVLegalizerInfo.cpp. 283 extern bool isTypeFoldingSupported(unsigned Opcode); 284 285 bool SPIRVInstructionSelector::select(MachineInstr &I) { 286 assert(I.getParent() && "Instruction should be in a basic block!"); 287 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 288 289 Register Opcode = I.getOpcode(); 290 // If it's not a GMIR instruction, we've selected it already. 291 if (!isPreISelGenericOpcode(Opcode)) { 292 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 293 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); 294 if (isTypeFoldingSupported(Def->getOpcode())) { 295 bool Res = selectImpl(I, *CoverageInfo); 296 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 297 if (Res) 298 return Res; 299 } 300 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); 301 I.removeFromParent(); 302 return true; 303 } else if (I.getNumDefs() == 1) { 304 // Make all vregs 32 bits (for SPIR-V IDs). 305 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); 306 } 307 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 308 } 309 310 if (I.getNumOperands() != I.getNumExplicitOperands()) { 311 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 312 return false; 313 } 314 315 // Common code for getting return reg+type, and removing selected instr 316 // from parent occurs here. Instr-specific selection happens in spvSelect(). 317 bool HasDefs = I.getNumDefs() > 0; 318 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 319 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 320 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 321 if (spvSelect(ResVReg, ResType, I)) { 322 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). 323 for (unsigned i = 0; i < I.getNumDefs(); ++i) 324 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32)); 325 I.removeFromParent(); 326 return true; 327 } 328 return false; 329 } 330 331 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 332 const SPIRVType *ResType, 333 MachineInstr &I) const { 334 const unsigned Opcode = I.getOpcode(); 335 if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT) 336 return selectImpl(I, *CoverageInfo); 337 switch (Opcode) { 338 case TargetOpcode::G_CONSTANT: 339 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 340 I); 341 case TargetOpcode::G_GLOBAL_VALUE: 342 return selectGlobalValue(ResVReg, I); 343 case TargetOpcode::G_IMPLICIT_DEF: 344 return selectOpUndef(ResVReg, ResType, I); 345 case TargetOpcode::G_FREEZE: 346 return selectFreeze(ResVReg, ResType, I); 347 348 case TargetOpcode::G_INTRINSIC: 349 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 350 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 351 return selectIntrinsic(ResVReg, ResType, I); 352 case TargetOpcode::G_BITREVERSE: 353 return selectBitreverse(ResVReg, ResType, I); 354 355 case TargetOpcode::G_BUILD_VECTOR: 356 return selectConstVector(ResVReg, ResType, I); 357 case TargetOpcode::G_SPLAT_VECTOR: 358 return selectSplatVector(ResVReg, ResType, I); 359 360 case TargetOpcode::G_SHUFFLE_VECTOR: { 361 MachineBasicBlock &BB = *I.getParent(); 362 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 363 .addDef(ResVReg) 364 .addUse(GR.getSPIRVTypeID(ResType)) 365 .addUse(I.getOperand(1).getReg()) 366 .addUse(I.getOperand(2).getReg()); 367 for (auto V : I.getOperand(3).getShuffleMask()) 368 MIB.addImm(V); 369 return MIB.constrainAllUses(TII, TRI, RBI); 370 } 371 case TargetOpcode::G_MEMMOVE: 372 case TargetOpcode::G_MEMCPY: 373 case TargetOpcode::G_MEMSET: 374 return selectMemOperation(ResVReg, I); 375 376 case TargetOpcode::G_ICMP: 377 return selectICmp(ResVReg, ResType, I); 378 case TargetOpcode::G_FCMP: 379 return selectFCmp(ResVReg, ResType, I); 380 381 case TargetOpcode::G_FRAME_INDEX: 382 return selectFrameIndex(ResVReg, ResType, I); 383 384 case TargetOpcode::G_LOAD: 385 return selectLoad(ResVReg, ResType, I); 386 case TargetOpcode::G_STORE: 387 return selectStore(I); 388 389 case TargetOpcode::G_BR: 390 return selectBranch(I); 391 case TargetOpcode::G_BRCOND: 392 return selectBranchCond(I); 393 394 case TargetOpcode::G_PHI: 395 return selectPhi(ResVReg, ResType, I); 396 397 case TargetOpcode::G_FPTOSI: 398 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 399 case TargetOpcode::G_FPTOUI: 400 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 401 402 case TargetOpcode::G_SITOFP: 403 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 404 case TargetOpcode::G_UITOFP: 405 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 406 407 case TargetOpcode::G_CTPOP: 408 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 409 case TargetOpcode::G_SMIN: 410 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 411 case TargetOpcode::G_UMIN: 412 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 413 414 case TargetOpcode::G_SMAX: 415 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 416 case TargetOpcode::G_UMAX: 417 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 418 419 case TargetOpcode::G_FMA: 420 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 421 422 case TargetOpcode::G_FPOW: 423 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 424 case TargetOpcode::G_FPOWI: 425 return selectExtInst(ResVReg, ResType, I, CL::pown); 426 427 case TargetOpcode::G_FEXP: 428 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 429 case TargetOpcode::G_FEXP2: 430 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 431 432 case TargetOpcode::G_FLOG: 433 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 434 case TargetOpcode::G_FLOG2: 435 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 436 case TargetOpcode::G_FLOG10: 437 return selectLog10(ResVReg, ResType, I); 438 439 case TargetOpcode::G_FABS: 440 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 441 case TargetOpcode::G_ABS: 442 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 443 444 case TargetOpcode::G_FMINNUM: 445 case TargetOpcode::G_FMINIMUM: 446 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 447 case TargetOpcode::G_FMAXNUM: 448 case TargetOpcode::G_FMAXIMUM: 449 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 450 451 case TargetOpcode::G_FCOPYSIGN: 452 return selectExtInst(ResVReg, ResType, I, CL::copysign); 453 454 case TargetOpcode::G_FCEIL: 455 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 456 case TargetOpcode::G_FFLOOR: 457 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 458 459 case TargetOpcode::G_FCOS: 460 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 461 case TargetOpcode::G_FSIN: 462 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 463 464 case TargetOpcode::G_FSQRT: 465 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 466 467 case TargetOpcode::G_CTTZ: 468 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 469 return selectExtInst(ResVReg, ResType, I, CL::ctz); 470 case TargetOpcode::G_CTLZ: 471 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 472 return selectExtInst(ResVReg, ResType, I, CL::clz); 473 474 case TargetOpcode::G_INTRINSIC_ROUND: 475 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 476 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 477 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 478 case TargetOpcode::G_INTRINSIC_TRUNC: 479 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 480 case TargetOpcode::G_FRINT: 481 case TargetOpcode::G_FNEARBYINT: 482 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 483 484 case TargetOpcode::G_SMULH: 485 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 486 case TargetOpcode::G_UMULH: 487 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 488 489 case TargetOpcode::G_SEXT: 490 return selectExt(ResVReg, ResType, I, true); 491 case TargetOpcode::G_ANYEXT: 492 case TargetOpcode::G_ZEXT: 493 return selectExt(ResVReg, ResType, I, false); 494 case TargetOpcode::G_TRUNC: 495 return selectTrunc(ResVReg, ResType, I); 496 case TargetOpcode::G_FPTRUNC: 497 case TargetOpcode::G_FPEXT: 498 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 499 500 case TargetOpcode::G_PTRTOINT: 501 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 502 case TargetOpcode::G_INTTOPTR: 503 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 504 case TargetOpcode::G_BITCAST: 505 return selectBitcast(ResVReg, ResType, I); 506 case TargetOpcode::G_ADDRSPACE_CAST: 507 return selectAddrSpaceCast(ResVReg, ResType, I); 508 case TargetOpcode::G_PTR_ADD: { 509 // Currently, we get G_PTR_ADD only as a result of translating 510 // global variables, initialized with constant expressions like GV + Const 511 // (see test opencl/basic/progvar_prog_scope_init.ll). 512 // TODO: extend the handler once we have other cases. 513 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 514 Register GV = I.getOperand(1).getReg(); 515 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 516 (void)II; 517 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 518 (*II).getOpcode() == TargetOpcode::COPY || 519 (*II).getOpcode() == SPIRV::OpVariable) && 520 isImm(I.getOperand(2), MRI)); 521 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 522 MachineBasicBlock &BB = *I.getParent(); 523 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 524 .addDef(ResVReg) 525 .addUse(GR.getSPIRVTypeID(ResType)) 526 .addImm(static_cast<uint32_t>( 527 SPIRV::Opcode::InBoundsPtrAccessChain)) 528 .addUse(GV) 529 .addUse(Idx) 530 .addUse(I.getOperand(2).getReg()); 531 return MIB.constrainAllUses(TII, TRI, RBI); 532 } 533 534 case TargetOpcode::G_ATOMICRMW_OR: 535 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 536 case TargetOpcode::G_ATOMICRMW_ADD: 537 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 538 case TargetOpcode::G_ATOMICRMW_AND: 539 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 540 case TargetOpcode::G_ATOMICRMW_MAX: 541 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 542 case TargetOpcode::G_ATOMICRMW_MIN: 543 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 544 case TargetOpcode::G_ATOMICRMW_SUB: 545 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 546 case TargetOpcode::G_ATOMICRMW_XOR: 547 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 548 case TargetOpcode::G_ATOMICRMW_UMAX: 549 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 550 case TargetOpcode::G_ATOMICRMW_UMIN: 551 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 552 case TargetOpcode::G_ATOMICRMW_XCHG: 553 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 554 case TargetOpcode::G_ATOMIC_CMPXCHG: 555 return selectAtomicCmpXchg(ResVReg, ResType, I); 556 557 case TargetOpcode::G_ATOMICRMW_FADD: 558 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 559 case TargetOpcode::G_ATOMICRMW_FSUB: 560 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 561 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 562 SPIRV::OpFNegate); 563 case TargetOpcode::G_ATOMICRMW_FMIN: 564 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 565 case TargetOpcode::G_ATOMICRMW_FMAX: 566 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 567 568 case TargetOpcode::G_FENCE: 569 return selectFence(I); 570 571 case TargetOpcode::G_STACKSAVE: 572 return selectStackSave(ResVReg, ResType, I); 573 case TargetOpcode::G_STACKRESTORE: 574 return selectStackRestore(I); 575 576 case TargetOpcode::G_UNMERGE_VALUES: 577 return selectUnmergeValues(I); 578 579 default: 580 return false; 581 } 582 } 583 584 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 585 const SPIRVType *ResType, 586 MachineInstr &I, 587 CL::OpenCLExtInst CLInst) const { 588 return selectExtInst(ResVReg, ResType, I, 589 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 590 } 591 592 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 593 const SPIRVType *ResType, 594 MachineInstr &I, 595 CL::OpenCLExtInst CLInst, 596 GL::GLSLExtInst GLInst) const { 597 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 598 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 599 return selectExtInst(ResVReg, ResType, I, ExtInsts); 600 } 601 602 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 603 const SPIRVType *ResType, 604 MachineInstr &I, 605 const ExtInstList &Insts) const { 606 607 for (const auto &Ex : Insts) { 608 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 609 uint32_t Opcode = Ex.second; 610 if (STI.canUseExtInstSet(Set)) { 611 MachineBasicBlock &BB = *I.getParent(); 612 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 613 .addDef(ResVReg) 614 .addUse(GR.getSPIRVTypeID(ResType)) 615 .addImm(static_cast<uint32_t>(Set)) 616 .addImm(Opcode); 617 const unsigned NumOps = I.getNumOperands(); 618 for (unsigned i = 1; i < NumOps; ++i) 619 MIB.add(I.getOperand(i)); 620 return MIB.constrainAllUses(TII, TRI, RBI); 621 } 622 } 623 return false; 624 } 625 626 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 627 const SPIRVType *ResType, 628 MachineInstr &I, 629 Register SrcReg, 630 unsigned Opcode) const { 631 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 632 .addDef(ResVReg) 633 .addUse(GR.getSPIRVTypeID(ResType)) 634 .addUse(SrcReg) 635 .constrainAllUses(TII, TRI, RBI); 636 } 637 638 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 639 const SPIRVType *ResType, 640 MachineInstr &I, 641 unsigned Opcode) const { 642 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 643 Opcode); 644 } 645 646 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 647 const SPIRVType *ResType, 648 MachineInstr &I) const { 649 Register OpReg = I.getOperand(1).getReg(); 650 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 651 if (!GR.isBitcastCompatible(ResType, OpType)) 652 report_fatal_error("incompatible result and operand types in a bitcast"); 653 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 654 } 655 656 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord, 657 SPIRVMachineModuleInfo *MMI) { 658 if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID) 659 return SPIRV::Scope::Invocation; 660 else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID) 661 return SPIRV::Scope::Device; 662 else if (Ord == MMI->WorkGroupSSID) 663 return SPIRV::Scope::Workgroup; 664 else if (Ord == MMI->AllSVMDevicesSSID) 665 return SPIRV::Scope::CrossDevice; 666 else if (Ord == MMI->SubGroupSSID) 667 return SPIRV::Scope::Subgroup; 668 else 669 // OpenCL approach is: "The functions that do not have memory_scope argument 670 // have the same semantics as the corresponding functions with the 671 // memory_scope argument set to memory_scope_device." See ref.: // 672 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions 673 // In our case if the scope is unknown, assuming that SPIR-V code is to be 674 // consumed in an OpenCL environment, we use the same approach and set the 675 // scope to memory_scope_device. 676 return SPIRV::Scope::Device; 677 } 678 679 static void addMemoryOperands(MachineMemOperand *MemOp, 680 MachineInstrBuilder &MIB) { 681 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 682 if (MemOp->isVolatile()) 683 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 684 if (MemOp->isNonTemporal()) 685 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 686 if (MemOp->getAlign().value()) 687 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 688 689 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 690 MIB.addImm(SpvMemOp); 691 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 692 MIB.addImm(MemOp->getAlign().value()); 693 } 694 } 695 696 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 697 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 698 if (Flags & MachineMemOperand::Flags::MOVolatile) 699 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 700 if (Flags & MachineMemOperand::Flags::MONonTemporal) 701 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 702 703 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 704 MIB.addImm(SpvMemOp); 705 } 706 707 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 708 const SPIRVType *ResType, 709 MachineInstr &I) const { 710 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 711 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 712 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 713 .addDef(ResVReg) 714 .addUse(GR.getSPIRVTypeID(ResType)) 715 .addUse(Ptr); 716 if (!I.getNumMemOperands()) { 717 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 718 I.getOpcode() == 719 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 720 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 721 } else { 722 addMemoryOperands(*I.memoperands_begin(), MIB); 723 } 724 return MIB.constrainAllUses(TII, TRI, RBI); 725 } 726 727 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 728 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 729 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 730 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 731 MachineBasicBlock &BB = *I.getParent(); 732 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 733 .addUse(Ptr) 734 .addUse(StoreVal); 735 if (!I.getNumMemOperands()) { 736 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 737 I.getOpcode() == 738 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 739 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 740 } else { 741 addMemoryOperands(*I.memoperands_begin(), MIB); 742 } 743 return MIB.constrainAllUses(TII, TRI, RBI); 744 } 745 746 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 747 const SPIRVType *ResType, 748 MachineInstr &I) const { 749 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 750 report_fatal_error( 751 "llvm.stacksave intrinsic: this instruction requires the following " 752 "SPIR-V extension: SPV_INTEL_variable_length_array", 753 false); 754 MachineBasicBlock &BB = *I.getParent(); 755 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 756 .addDef(ResVReg) 757 .addUse(GR.getSPIRVTypeID(ResType)) 758 .constrainAllUses(TII, TRI, RBI); 759 } 760 761 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 762 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 763 report_fatal_error( 764 "llvm.stackrestore intrinsic: this instruction requires the following " 765 "SPIR-V extension: SPV_INTEL_variable_length_array", 766 false); 767 if (!I.getOperand(0).isReg()) 768 return false; 769 MachineBasicBlock &BB = *I.getParent(); 770 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 771 .addUse(I.getOperand(0).getReg()) 772 .constrainAllUses(TII, TRI, RBI); 773 } 774 775 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 776 MachineInstr &I) const { 777 MachineBasicBlock &BB = *I.getParent(); 778 Register SrcReg = I.getOperand(1).getReg(); 779 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 780 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 781 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 782 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 783 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 784 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 785 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); 786 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 787 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 788 // TODO: check if we have such GV, add init, use buildGlobalVariable. 789 Function &CurFunction = GR.CurMF->getFunction(); 790 Type *LLVMArrTy = 791 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 792 // Module takes ownership of the global var. 793 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 794 true, GlobalValue::InternalLinkage, 795 Constant::getNullValue(LLVMArrTy)); 796 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 797 GR.add(GV, GR.CurMF, VarReg); 798 799 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 800 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 801 .addDef(VarReg) 802 .addUse(GR.getSPIRVTypeID(VarTy)) 803 .addImm(SPIRV::StorageClass::UniformConstant) 804 .addUse(Const) 805 .constrainAllUses(TII, TRI, RBI); 806 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 807 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 808 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 809 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 810 } 811 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 812 .addUse(I.getOperand(0).getReg()) 813 .addUse(SrcReg) 814 .addUse(I.getOperand(2).getReg()); 815 if (I.getNumMemOperands()) 816 addMemoryOperands(*I.memoperands_begin(), MIB); 817 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 818 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 819 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 820 .addUse(MIB->getOperand(0).getReg()); 821 return Result; 822 } 823 824 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 825 const SPIRVType *ResType, 826 MachineInstr &I, 827 unsigned NewOpcode, 828 unsigned NegateOpcode) const { 829 assert(I.hasOneMemOperand()); 830 const MachineMemOperand *MemOp = *I.memoperands_begin(); 831 uint32_t Scope = 832 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); 833 Register ScopeReg = buildI32Constant(Scope, I); 834 835 Register Ptr = I.getOperand(1).getReg(); 836 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 837 // auto ScSem = 838 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 839 AtomicOrdering AO = MemOp->getSuccessOrdering(); 840 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 841 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 842 843 bool Result = false; 844 Register ValueReg = I.getOperand(2).getReg(); 845 if (NegateOpcode != 0) { 846 // Translation with negative value operand is requested 847 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 848 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 849 ValueReg = TmpReg; 850 } 851 852 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 853 .addDef(ResVReg) 854 .addUse(GR.getSPIRVTypeID(ResType)) 855 .addUse(Ptr) 856 .addUse(ScopeReg) 857 .addUse(MemSemReg) 858 .addUse(ValueReg) 859 .constrainAllUses(TII, TRI, RBI); 860 return Result; 861 } 862 863 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 864 unsigned ArgI = I.getNumOperands() - 1; 865 Register SrcReg = 866 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 867 SPIRVType *DefType = 868 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 869 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 870 report_fatal_error( 871 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 872 873 SPIRVType *ScalarType = 874 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 875 MachineBasicBlock &BB = *I.getParent(); 876 bool Res = false; 877 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 878 Register ResVReg = I.getOperand(i).getReg(); 879 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 880 if (!ResType) { 881 // There was no "assign type" actions, let's fix this now 882 ResType = ScalarType; 883 MRI->setRegClass(ResVReg, &SPIRV::IDRegClass); 884 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 885 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 886 } 887 auto MIB = 888 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 889 .addDef(ResVReg) 890 .addUse(GR.getSPIRVTypeID(ResType)) 891 .addUse(SrcReg) 892 .addImm(static_cast<int64_t>(i)); 893 Res |= MIB.constrainAllUses(TII, TRI, RBI); 894 } 895 return Res; 896 } 897 898 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 899 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 900 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 901 Register MemSemReg = buildI32Constant(MemSem, I); 902 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 903 uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI)); 904 Register ScopeReg = buildI32Constant(Scope, I); 905 MachineBasicBlock &BB = *I.getParent(); 906 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 907 .addUse(ScopeReg) 908 .addUse(MemSemReg) 909 .constrainAllUses(TII, TRI, RBI); 910 } 911 912 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 913 const SPIRVType *ResType, 914 MachineInstr &I) const { 915 Register ScopeReg; 916 Register MemSemEqReg; 917 Register MemSemNeqReg; 918 Register Ptr = I.getOperand(2).getReg(); 919 if (!isa<GIntrinsic>(I)) { 920 assert(I.hasOneMemOperand()); 921 const MachineMemOperand *MemOp = *I.memoperands_begin(); 922 unsigned Scope = 923 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); 924 ScopeReg = buildI32Constant(Scope, I); 925 926 unsigned ScSem = static_cast<uint32_t>( 927 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 928 AtomicOrdering AO = MemOp->getSuccessOrdering(); 929 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 930 MemSemEqReg = buildI32Constant(MemSemEq, I); 931 AtomicOrdering FO = MemOp->getFailureOrdering(); 932 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 933 MemSemNeqReg = 934 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 935 } else { 936 ScopeReg = I.getOperand(5).getReg(); 937 MemSemEqReg = I.getOperand(6).getReg(); 938 MemSemNeqReg = I.getOperand(7).getReg(); 939 } 940 941 Register Cmp = I.getOperand(3).getReg(); 942 Register Val = I.getOperand(4).getReg(); 943 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 944 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); 945 const DebugLoc &DL = I.getDebugLoc(); 946 bool Result = 947 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 948 .addDef(ACmpRes) 949 .addUse(GR.getSPIRVTypeID(SpvValTy)) 950 .addUse(Ptr) 951 .addUse(ScopeReg) 952 .addUse(MemSemEqReg) 953 .addUse(MemSemNeqReg) 954 .addUse(Val) 955 .addUse(Cmp) 956 .constrainAllUses(TII, TRI, RBI); 957 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 958 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 959 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 960 .addDef(CmpSuccReg) 961 .addUse(GR.getSPIRVTypeID(BoolTy)) 962 .addUse(ACmpRes) 963 .addUse(Cmp) 964 .constrainAllUses(TII, TRI, RBI); 965 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 966 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 967 .addDef(TmpReg) 968 .addUse(GR.getSPIRVTypeID(ResType)) 969 .addUse(ACmpRes) 970 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 971 .addImm(0) 972 .constrainAllUses(TII, TRI, RBI); 973 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 974 .addDef(ResVReg) 975 .addUse(GR.getSPIRVTypeID(ResType)) 976 .addUse(CmpSuccReg) 977 .addUse(TmpReg) 978 .addImm(1) 979 .constrainAllUses(TII, TRI, RBI); 980 return Result; 981 } 982 983 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 984 switch (SC) { 985 case SPIRV::StorageClass::Workgroup: 986 case SPIRV::StorageClass::CrossWorkgroup: 987 case SPIRV::StorageClass::Function: 988 return true; 989 default: 990 return false; 991 } 992 } 993 994 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 995 switch (SC) { 996 case SPIRV::StorageClass::DeviceOnlyINTEL: 997 case SPIRV::StorageClass::HostOnlyINTEL: 998 return true; 999 default: 1000 return false; 1001 } 1002 } 1003 1004 // In SPIR-V address space casting can only happen to and from the Generic 1005 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 1006 // pointers to and from Generic pointers. As such, we can convert e.g. from 1007 // Workgroup to Function by going via a Generic pointer as an intermediary. All 1008 // other combinations can only be done by a bitcast, and are probably not safe. 1009 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 1010 const SPIRVType *ResType, 1011 MachineInstr &I) const { 1012 // If the AddrSpaceCast user is single and in OpConstantComposite or 1013 // OpVariable, we should select OpSpecConstantOp. 1014 auto UIs = MRI->use_instructions(ResVReg); 1015 if (!UIs.empty() && ++UIs.begin() == UIs.end() && 1016 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || 1017 UIs.begin()->getOpcode() == SPIRV::OpVariable || 1018 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { 1019 Register NewReg = I.getOperand(1).getReg(); 1020 MachineBasicBlock &BB = *I.getParent(); 1021 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1022 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, 1023 SPIRV::StorageClass::Generic); 1024 bool Result = 1025 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 1026 .addDef(ResVReg) 1027 .addUse(GR.getSPIRVTypeID(ResType)) 1028 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) 1029 .addUse(NewReg) 1030 .constrainAllUses(TII, TRI, RBI); 1031 return Result; 1032 } 1033 Register SrcPtr = I.getOperand(1).getReg(); 1034 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1035 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); 1036 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); 1037 1038 // don't generate a cast between identical storage classes 1039 if (SrcSC == DstSC) 1040 return true; 1041 1042 // Casting from an eligible pointer to Generic. 1043 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1044 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1045 // Casting from Generic to an eligible pointer. 1046 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1047 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1048 // Casting between 2 eligible pointers using Generic as an intermediary. 1049 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1050 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1051 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1052 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); 1053 MachineBasicBlock &BB = *I.getParent(); 1054 const DebugLoc &DL = I.getDebugLoc(); 1055 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1056 .addDef(Tmp) 1057 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1058 .addUse(SrcPtr) 1059 .constrainAllUses(TII, TRI, RBI); 1060 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1061 .addDef(ResVReg) 1062 .addUse(GR.getSPIRVTypeID(ResType)) 1063 .addUse(Tmp) 1064 .constrainAllUses(TII, TRI, RBI); 1065 } 1066 1067 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1068 // be applied 1069 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1070 return selectUnOp(ResVReg, ResType, I, 1071 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1072 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1073 return selectUnOp(ResVReg, ResType, I, 1074 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1075 1076 // TODO Should this case just be disallowed completely? 1077 // We're casting 2 other arbitrary address spaces, so have to bitcast. 1078 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1079 } 1080 1081 static unsigned getFCmpOpcode(unsigned PredNum) { 1082 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1083 switch (Pred) { 1084 case CmpInst::FCMP_OEQ: 1085 return SPIRV::OpFOrdEqual; 1086 case CmpInst::FCMP_OGE: 1087 return SPIRV::OpFOrdGreaterThanEqual; 1088 case CmpInst::FCMP_OGT: 1089 return SPIRV::OpFOrdGreaterThan; 1090 case CmpInst::FCMP_OLE: 1091 return SPIRV::OpFOrdLessThanEqual; 1092 case CmpInst::FCMP_OLT: 1093 return SPIRV::OpFOrdLessThan; 1094 case CmpInst::FCMP_ONE: 1095 return SPIRV::OpFOrdNotEqual; 1096 case CmpInst::FCMP_ORD: 1097 return SPIRV::OpOrdered; 1098 case CmpInst::FCMP_UEQ: 1099 return SPIRV::OpFUnordEqual; 1100 case CmpInst::FCMP_UGE: 1101 return SPIRV::OpFUnordGreaterThanEqual; 1102 case CmpInst::FCMP_UGT: 1103 return SPIRV::OpFUnordGreaterThan; 1104 case CmpInst::FCMP_ULE: 1105 return SPIRV::OpFUnordLessThanEqual; 1106 case CmpInst::FCMP_ULT: 1107 return SPIRV::OpFUnordLessThan; 1108 case CmpInst::FCMP_UNE: 1109 return SPIRV::OpFUnordNotEqual; 1110 case CmpInst::FCMP_UNO: 1111 return SPIRV::OpUnordered; 1112 default: 1113 llvm_unreachable("Unknown predicate type for FCmp"); 1114 } 1115 } 1116 1117 static unsigned getICmpOpcode(unsigned PredNum) { 1118 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1119 switch (Pred) { 1120 case CmpInst::ICMP_EQ: 1121 return SPIRV::OpIEqual; 1122 case CmpInst::ICMP_NE: 1123 return SPIRV::OpINotEqual; 1124 case CmpInst::ICMP_SGE: 1125 return SPIRV::OpSGreaterThanEqual; 1126 case CmpInst::ICMP_SGT: 1127 return SPIRV::OpSGreaterThan; 1128 case CmpInst::ICMP_SLE: 1129 return SPIRV::OpSLessThanEqual; 1130 case CmpInst::ICMP_SLT: 1131 return SPIRV::OpSLessThan; 1132 case CmpInst::ICMP_UGE: 1133 return SPIRV::OpUGreaterThanEqual; 1134 case CmpInst::ICMP_UGT: 1135 return SPIRV::OpUGreaterThan; 1136 case CmpInst::ICMP_ULE: 1137 return SPIRV::OpULessThanEqual; 1138 case CmpInst::ICMP_ULT: 1139 return SPIRV::OpULessThan; 1140 default: 1141 llvm_unreachable("Unknown predicate type for ICmp"); 1142 } 1143 } 1144 1145 static unsigned getPtrCmpOpcode(unsigned Pred) { 1146 switch (static_cast<CmpInst::Predicate>(Pred)) { 1147 case CmpInst::ICMP_EQ: 1148 return SPIRV::OpPtrEqual; 1149 case CmpInst::ICMP_NE: 1150 return SPIRV::OpPtrNotEqual; 1151 default: 1152 llvm_unreachable("Unknown predicate type for pointer comparison"); 1153 } 1154 } 1155 1156 // Return the logical operation, or abort if none exists. 1157 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1158 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1159 switch (Pred) { 1160 case CmpInst::ICMP_EQ: 1161 return SPIRV::OpLogicalEqual; 1162 case CmpInst::ICMP_NE: 1163 return SPIRV::OpLogicalNotEqual; 1164 default: 1165 llvm_unreachable("Unknown predicate type for Bool comparison"); 1166 } 1167 } 1168 1169 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, 1170 const SPIRVType *ResType, 1171 MachineInstr &I, 1172 unsigned OpAnyOrAll) const { 1173 assert(I.getNumOperands() == 3); 1174 assert(I.getOperand(2).isReg()); 1175 MachineBasicBlock &BB = *I.getParent(); 1176 Register InputRegister = I.getOperand(2).getReg(); 1177 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); 1178 1179 if (!InputType) 1180 report_fatal_error("Input Type could not be determined."); 1181 1182 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); 1183 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; 1184 if (IsBoolTy && !IsVectorTy) { 1185 assert(ResVReg == I.getOperand(0).getReg()); 1186 return BuildMI(*I.getParent(), I, I.getDebugLoc(), 1187 TII.get(TargetOpcode::COPY)) 1188 .addDef(ResVReg) 1189 .addUse(InputRegister) 1190 .constrainAllUses(TII, TRI, RBI); 1191 } 1192 1193 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); 1194 unsigned SpirvNotEqualId = 1195 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; 1196 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); 1197 SPIRVType *SpvBoolTy = SpvBoolScalarTy; 1198 Register NotEqualReg = ResVReg; 1199 1200 if (IsVectorTy) { 1201 NotEqualReg = IsBoolTy ? InputRegister 1202 : MRI->createVirtualRegister(&SPIRV::IDRegClass); 1203 const unsigned NumElts = InputType->getOperand(2).getImm(); 1204 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); 1205 } 1206 1207 if (!IsBoolTy) { 1208 Register ConstZeroReg = 1209 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); 1210 1211 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) 1212 .addDef(NotEqualReg) 1213 .addUse(GR.getSPIRVTypeID(SpvBoolTy)) 1214 .addUse(InputRegister) 1215 .addUse(ConstZeroReg) 1216 .constrainAllUses(TII, TRI, RBI); 1217 } 1218 1219 if (!IsVectorTy) 1220 return true; 1221 1222 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) 1223 .addDef(ResVReg) 1224 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) 1225 .addUse(NotEqualReg) 1226 .constrainAllUses(TII, TRI, RBI); 1227 } 1228 1229 bool SPIRVInstructionSelector::selectAll(Register ResVReg, 1230 const SPIRVType *ResType, 1231 MachineInstr &I) const { 1232 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); 1233 } 1234 1235 bool SPIRVInstructionSelector::selectAny(Register ResVReg, 1236 const SPIRVType *ResType, 1237 MachineInstr &I) const { 1238 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); 1239 } 1240 1241 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1242 const SPIRVType *ResType, 1243 MachineInstr &I) const { 1244 MachineBasicBlock &BB = *I.getParent(); 1245 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1246 .addDef(ResVReg) 1247 .addUse(GR.getSPIRVTypeID(ResType)) 1248 .addUse(I.getOperand(1).getReg()) 1249 .constrainAllUses(TII, TRI, RBI); 1250 } 1251 1252 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1253 const SPIRVType *ResType, 1254 MachineInstr &I) const { 1255 // There is no way to implement `freeze` correctly without support on SPIR-V 1256 // standard side, but we may at least address a simple (static) case when 1257 // undef/poison value presence is obvious. The main benefit of even 1258 // incomplete `freeze` support is preventing of translation from crashing due 1259 // to lack of support on legalization and instruction selection steps. 1260 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1261 return false; 1262 Register OpReg = I.getOperand(1).getReg(); 1263 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1264 Register Reg; 1265 switch (Def->getOpcode()) { 1266 case SPIRV::ASSIGN_TYPE: 1267 if (MachineInstr *AssignToDef = 1268 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1269 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1270 Reg = Def->getOperand(2).getReg(); 1271 } 1272 break; 1273 case SPIRV::OpUndef: 1274 Reg = Def->getOperand(1).getReg(); 1275 break; 1276 } 1277 unsigned DestOpCode; 1278 if (Reg.isValid()) { 1279 DestOpCode = SPIRV::OpConstantNull; 1280 } else { 1281 DestOpCode = TargetOpcode::COPY; 1282 Reg = OpReg; 1283 } 1284 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1285 .addDef(I.getOperand(0).getReg()) 1286 .addUse(Reg) 1287 .constrainAllUses(TII, TRI, RBI); 1288 } 1289 return false; 1290 } 1291 1292 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, 1293 const SPIRVType *ResType, 1294 MachineInstr &I) const { 1295 // TODO: only const case is supported for now. 1296 assert(std::all_of( 1297 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { 1298 if (MO.isDef()) 1299 return true; 1300 if (!MO.isReg()) 1301 return false; 1302 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); 1303 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 1304 ConstTy->getOperand(1).isReg()); 1305 Register ConstReg = ConstTy->getOperand(1).getReg(); 1306 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 1307 assert(Const); 1308 return (Const->getOpcode() == TargetOpcode::G_CONSTANT || 1309 Const->getOpcode() == TargetOpcode::G_FCONSTANT); 1310 })); 1311 1312 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1313 TII.get(SPIRV::OpConstantComposite)) 1314 .addDef(ResVReg) 1315 .addUse(GR.getSPIRVTypeID(ResType)); 1316 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 1317 MIB.addUse(I.getOperand(i).getReg()); 1318 return MIB.constrainAllUses(TII, TRI, RBI); 1319 } 1320 1321 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1322 const SPIRVType *ResType) { 1323 Register OpReg = ResType->getOperand(2).getReg(); 1324 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1325 if (!OpDef) 1326 return 0; 1327 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1328 OpDef->getOperand(1).isReg()) { 1329 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1330 OpDef = RefDef; 1331 } 1332 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 1333 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 1334 : 0; 1335 return N; 1336 } 1337 1338 // Return true if the type represents a constant register 1339 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { 1340 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1341 OpDef->getOperand(1).isReg()) { 1342 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1343 OpDef = RefDef; 1344 } 1345 return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || 1346 OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; 1347 } 1348 1349 // Return true if the virtual register represents a constant 1350 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 1351 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 1352 return isConstReg(MRI, OpDef); 1353 return false; 1354 } 1355 1356 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 1357 const SPIRVType *ResType, 1358 MachineInstr &I) const { 1359 unsigned N = 0; 1360 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1361 N = GR.getScalarOrVectorComponentCount(ResType); 1362 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1363 N = getArrayComponentCount(MRI, ResType); 1364 else 1365 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 1366 1367 unsigned OpIdx = I.getNumExplicitDefs(); 1368 if (!I.getOperand(OpIdx).isReg()) 1369 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 1370 1371 // check if we may construct a constant vector 1372 Register OpReg = I.getOperand(OpIdx).getReg(); 1373 bool IsConst = isConstReg(MRI, OpReg); 1374 1375 if (!IsConst && N < 2) 1376 report_fatal_error( 1377 "There must be at least two constituent operands in a vector"); 1378 1379 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1380 TII.get(IsConst ? SPIRV::OpConstantComposite 1381 : SPIRV::OpCompositeConstruct)) 1382 .addDef(ResVReg) 1383 .addUse(GR.getSPIRVTypeID(ResType)); 1384 for (unsigned i = 0; i < N; ++i) 1385 MIB.addUse(OpReg); 1386 return MIB.constrainAllUses(TII, TRI, RBI); 1387 } 1388 1389 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 1390 const SPIRVType *ResType, 1391 unsigned CmpOpc, 1392 MachineInstr &I) const { 1393 Register Cmp0 = I.getOperand(2).getReg(); 1394 Register Cmp1 = I.getOperand(3).getReg(); 1395 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1396 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1397 "CMP operands should have the same type"); 1398 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1399 .addDef(ResVReg) 1400 .addUse(GR.getSPIRVTypeID(ResType)) 1401 .addUse(Cmp0) 1402 .addUse(Cmp1) 1403 .constrainAllUses(TII, TRI, RBI); 1404 } 1405 1406 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1407 const SPIRVType *ResType, 1408 MachineInstr &I) const { 1409 auto Pred = I.getOperand(1).getPredicate(); 1410 unsigned CmpOpc; 1411 1412 Register CmpOperand = I.getOperand(2).getReg(); 1413 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1414 CmpOpc = getPtrCmpOpcode(Pred); 1415 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1416 CmpOpc = getBoolCmpOpcode(Pred); 1417 else 1418 CmpOpc = getICmpOpcode(Pred); 1419 return selectCmp(ResVReg, ResType, CmpOpc, I); 1420 } 1421 1422 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, 1423 const MachineInstr &I, 1424 int OpIdx) const { 1425 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1426 "Expected G_FCONSTANT"); 1427 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1428 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1429 } 1430 1431 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1432 const MachineInstr &I, 1433 int OpIdx) const { 1434 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1435 "Expected G_CONSTANT"); 1436 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1437 } 1438 1439 Register 1440 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1441 const SPIRVType *ResType) const { 1442 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1443 const SPIRVType *SpvI32Ty = 1444 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1445 // Find a constant in DT or build a new one. 1446 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1447 Register NewReg = GR.find(ConstInt, GR.CurMF); 1448 if (!NewReg.isValid()) { 1449 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1450 GR.add(ConstInt, GR.CurMF, NewReg); 1451 MachineInstr *MI; 1452 MachineBasicBlock &BB = *I.getParent(); 1453 if (Val == 0) { 1454 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1455 .addDef(NewReg) 1456 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 1457 } else { 1458 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1459 .addDef(NewReg) 1460 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 1461 .addImm(APInt(32, Val).getZExtValue()); 1462 } 1463 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 1464 } 1465 return NewReg; 1466 } 1467 1468 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 1469 const SPIRVType *ResType, 1470 MachineInstr &I) const { 1471 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 1472 return selectCmp(ResVReg, ResType, CmpOp, I); 1473 } 1474 1475 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 1476 MachineInstr &I) const { 1477 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 1478 bool ZeroAsNull = STI.isOpenCLEnv(); 1479 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1480 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); 1481 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); 1482 } 1483 1484 static APFloat getZeroFP(const Type *LLVMFloatTy) { 1485 if (!LLVMFloatTy) 1486 return APFloat::getZero(APFloat::IEEEsingle()); 1487 switch (LLVMFloatTy->getScalarType()->getTypeID()) { 1488 case Type::HalfTyID: 1489 return APFloat::getZero(APFloat::IEEEhalf()); 1490 default: 1491 case Type::FloatTyID: 1492 return APFloat::getZero(APFloat::IEEEsingle()); 1493 case Type::DoubleTyID: 1494 return APFloat::getZero(APFloat::IEEEdouble()); 1495 } 1496 } 1497 1498 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, 1499 MachineInstr &I) const { 1500 // OpenCL uses nulls for Zero. In HLSL we don't use null constants. 1501 bool ZeroAsNull = STI.isOpenCLEnv(); 1502 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); 1503 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1504 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); 1505 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); 1506 } 1507 1508 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 1509 const SPIRVType *ResType, 1510 MachineInstr &I) const { 1511 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1512 APInt One = 1513 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 1514 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1515 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); 1516 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 1517 } 1518 1519 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 1520 const SPIRVType *ResType, 1521 MachineInstr &I, 1522 bool IsSigned) const { 1523 // To extend a bool, we need to use OpSelect between constants. 1524 Register ZeroReg = buildZerosVal(ResType, I); 1525 Register OneReg = buildOnesVal(IsSigned, ResType, I); 1526 bool IsScalarBool = 1527 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 1528 unsigned Opcode = 1529 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 1530 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1531 .addDef(ResVReg) 1532 .addUse(GR.getSPIRVTypeID(ResType)) 1533 .addUse(I.getOperand(1).getReg()) 1534 .addUse(OneReg) 1535 .addUse(ZeroReg) 1536 .constrainAllUses(TII, TRI, RBI); 1537 } 1538 1539 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 1540 const SPIRVType *ResType, 1541 MachineInstr &I, bool IsSigned, 1542 unsigned Opcode) const { 1543 Register SrcReg = I.getOperand(1).getReg(); 1544 // We can convert bool value directly to float type without OpConvert*ToF, 1545 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 1546 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 1547 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1548 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 1549 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 1550 const unsigned NumElts = ResType->getOperand(2).getImm(); 1551 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 1552 } 1553 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1554 selectSelect(SrcReg, TmpType, I, false); 1555 } 1556 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 1557 } 1558 1559 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 1560 const SPIRVType *ResType, 1561 MachineInstr &I, bool IsSigned) const { 1562 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) 1563 return selectSelect(ResVReg, ResType, I, IsSigned); 1564 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1565 return selectUnOp(ResVReg, ResType, I, Opcode); 1566 } 1567 1568 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 1569 Register ResVReg, 1570 MachineInstr &I, 1571 const SPIRVType *IntTy, 1572 const SPIRVType *BoolTy) const { 1573 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 1574 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1575 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 1576 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 1577 Register Zero = buildZerosVal(IntTy, I); 1578 Register One = buildOnesVal(false, IntTy, I); 1579 MachineBasicBlock &BB = *I.getParent(); 1580 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1581 .addDef(BitIntReg) 1582 .addUse(GR.getSPIRVTypeID(IntTy)) 1583 .addUse(IntReg) 1584 .addUse(One) 1585 .constrainAllUses(TII, TRI, RBI); 1586 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1587 .addDef(ResVReg) 1588 .addUse(GR.getSPIRVTypeID(BoolTy)) 1589 .addUse(BitIntReg) 1590 .addUse(Zero) 1591 .constrainAllUses(TII, TRI, RBI); 1592 } 1593 1594 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 1595 const SPIRVType *ResType, 1596 MachineInstr &I) const { 1597 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { 1598 Register IntReg = I.getOperand(1).getReg(); 1599 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 1600 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 1601 } 1602 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 1603 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1604 return selectUnOp(ResVReg, ResType, I, Opcode); 1605 } 1606 1607 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 1608 const SPIRVType *ResType, 1609 const APInt &Imm, 1610 MachineInstr &I) const { 1611 unsigned TyOpcode = ResType->getOpcode(); 1612 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 1613 MachineBasicBlock &BB = *I.getParent(); 1614 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 1615 Imm.isZero()) 1616 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1617 .addDef(ResVReg) 1618 .addUse(GR.getSPIRVTypeID(ResType)) 1619 .constrainAllUses(TII, TRI, RBI); 1620 if (TyOpcode == SPIRV::OpTypeInt) { 1621 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 1622 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 1623 if (Reg == ResVReg) 1624 return true; 1625 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1626 .addDef(ResVReg) 1627 .addUse(Reg) 1628 .constrainAllUses(TII, TRI, RBI); 1629 } 1630 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1631 .addDef(ResVReg) 1632 .addUse(GR.getSPIRVTypeID(ResType)); 1633 // <=32-bit integers should be caught by the sdag pattern. 1634 assert(Imm.getBitWidth() > 32); 1635 addNumImm(Imm, MIB); 1636 return MIB.constrainAllUses(TII, TRI, RBI); 1637 } 1638 1639 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 1640 const SPIRVType *ResType, 1641 MachineInstr &I) const { 1642 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1643 .addDef(ResVReg) 1644 .addUse(GR.getSPIRVTypeID(ResType)) 1645 .constrainAllUses(TII, TRI, RBI); 1646 } 1647 1648 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1649 assert(MO.isReg()); 1650 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1651 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) 1652 return false; 1653 assert(TypeInst->getOperand(1).isReg()); 1654 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1655 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 1656 } 1657 1658 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1659 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1660 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1661 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 1662 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 1663 } 1664 1665 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 1666 const SPIRVType *ResType, 1667 MachineInstr &I) const { 1668 MachineBasicBlock &BB = *I.getParent(); 1669 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 1670 .addDef(ResVReg) 1671 .addUse(GR.getSPIRVTypeID(ResType)) 1672 // object to insert 1673 .addUse(I.getOperand(3).getReg()) 1674 // composite to insert into 1675 .addUse(I.getOperand(2).getReg()); 1676 for (unsigned i = 4; i < I.getNumOperands(); i++) 1677 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1678 return MIB.constrainAllUses(TII, TRI, RBI); 1679 } 1680 1681 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 1682 const SPIRVType *ResType, 1683 MachineInstr &I) const { 1684 MachineBasicBlock &BB = *I.getParent(); 1685 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1686 .addDef(ResVReg) 1687 .addUse(GR.getSPIRVTypeID(ResType)) 1688 .addUse(I.getOperand(2).getReg()); 1689 for (unsigned i = 3; i < I.getNumOperands(); i++) 1690 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1691 return MIB.constrainAllUses(TII, TRI, RBI); 1692 } 1693 1694 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 1695 const SPIRVType *ResType, 1696 MachineInstr &I) const { 1697 if (isImm(I.getOperand(4), MRI)) 1698 return selectInsertVal(ResVReg, ResType, I); 1699 MachineBasicBlock &BB = *I.getParent(); 1700 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 1701 .addDef(ResVReg) 1702 .addUse(GR.getSPIRVTypeID(ResType)) 1703 .addUse(I.getOperand(2).getReg()) 1704 .addUse(I.getOperand(3).getReg()) 1705 .addUse(I.getOperand(4).getReg()) 1706 .constrainAllUses(TII, TRI, RBI); 1707 } 1708 1709 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 1710 const SPIRVType *ResType, 1711 MachineInstr &I) const { 1712 if (isImm(I.getOperand(3), MRI)) 1713 return selectExtractVal(ResVReg, ResType, I); 1714 MachineBasicBlock &BB = *I.getParent(); 1715 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 1716 .addDef(ResVReg) 1717 .addUse(GR.getSPIRVTypeID(ResType)) 1718 .addUse(I.getOperand(2).getReg()) 1719 .addUse(I.getOperand(3).getReg()) 1720 .constrainAllUses(TII, TRI, RBI); 1721 } 1722 1723 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 1724 const SPIRVType *ResType, 1725 MachineInstr &I) const { 1726 const bool IsGEPInBounds = I.getOperand(2).getImm(); 1727 1728 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 1729 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 1730 // we have to use Op[InBounds]AccessChain. 1731 const unsigned Opcode = STI.isVulkanEnv() 1732 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 1733 : SPIRV::OpAccessChain) 1734 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 1735 : SPIRV::OpPtrAccessChain); 1736 1737 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1738 .addDef(ResVReg) 1739 .addUse(GR.getSPIRVTypeID(ResType)) 1740 // Object to get a pointer to. 1741 .addUse(I.getOperand(3).getReg()); 1742 // Adding indices. 1743 const unsigned StartingIndex = 1744 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 1745 ? 5 1746 : 4; 1747 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 1748 Res.addUse(I.getOperand(i).getReg()); 1749 return Res.constrainAllUses(TII, TRI, RBI); 1750 } 1751 1752 // Maybe wrap a value into OpSpecConstantOp 1753 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 1754 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 1755 bool Result = true; 1756 unsigned Lim = I.getNumExplicitOperands(); 1757 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 1758 Register OpReg = I.getOperand(i).getReg(); 1759 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 1760 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 1761 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || 1762 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { 1763 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 1764 // by selectAddrSpaceCast() 1765 CompositeArgs.push_back(OpReg); 1766 continue; 1767 } 1768 MachineFunction *MF = I.getMF(); 1769 Register WrapReg = GR.find(OpDefine, MF); 1770 if (WrapReg.isValid()) { 1771 CompositeArgs.push_back(WrapReg); 1772 continue; 1773 } 1774 // Create a new register for the wrapper 1775 WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1776 GR.add(OpDefine, MF, WrapReg); 1777 CompositeArgs.push_back(WrapReg); 1778 // Decorate the wrapper register and generate a new instruction 1779 MRI->setType(WrapReg, LLT::pointer(0, 32)); 1780 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 1781 MachineBasicBlock &BB = *I.getParent(); 1782 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 1783 .addDef(WrapReg) 1784 .addUse(GR.getSPIRVTypeID(OpType)) 1785 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 1786 .addUse(OpReg) 1787 .constrainAllUses(TII, TRI, RBI); 1788 if (!Result) 1789 break; 1790 } 1791 return Result; 1792 } 1793 1794 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 1795 const SPIRVType *ResType, 1796 MachineInstr &I) const { 1797 MachineBasicBlock &BB = *I.getParent(); 1798 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 1799 switch (IID) { 1800 case Intrinsic::spv_load: 1801 return selectLoad(ResVReg, ResType, I); 1802 case Intrinsic::spv_store: 1803 return selectStore(I); 1804 case Intrinsic::spv_extractv: 1805 return selectExtractVal(ResVReg, ResType, I); 1806 case Intrinsic::spv_insertv: 1807 return selectInsertVal(ResVReg, ResType, I); 1808 case Intrinsic::spv_extractelt: 1809 return selectExtractElt(ResVReg, ResType, I); 1810 case Intrinsic::spv_insertelt: 1811 return selectInsertElt(ResVReg, ResType, I); 1812 case Intrinsic::spv_gep: 1813 return selectGEP(ResVReg, ResType, I); 1814 case Intrinsic::spv_unref_global: 1815 case Intrinsic::spv_init_global: { 1816 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 1817 MachineInstr *Init = I.getNumExplicitOperands() > 2 1818 ? MRI->getVRegDef(I.getOperand(2).getReg()) 1819 : nullptr; 1820 assert(MI); 1821 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 1822 } 1823 case Intrinsic::spv_undef: { 1824 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1825 .addDef(ResVReg) 1826 .addUse(GR.getSPIRVTypeID(ResType)); 1827 return MIB.constrainAllUses(TII, TRI, RBI); 1828 } 1829 case Intrinsic::spv_const_composite: { 1830 // If no values are attached, the composite is null constant. 1831 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 1832 // Select a proper instruction. 1833 unsigned Opcode = SPIRV::OpConstantNull; 1834 SmallVector<Register> CompositeArgs; 1835 if (!IsNull) { 1836 Opcode = SPIRV::OpConstantComposite; 1837 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 1838 return false; 1839 } 1840 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1841 .addDef(ResVReg) 1842 .addUse(GR.getSPIRVTypeID(ResType)); 1843 // skip type MD node we already used when generated assign.type for this 1844 if (!IsNull) { 1845 for (Register OpReg : CompositeArgs) 1846 MIB.addUse(OpReg); 1847 } 1848 return MIB.constrainAllUses(TII, TRI, RBI); 1849 } 1850 case Intrinsic::spv_assign_name: { 1851 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 1852 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 1853 for (unsigned i = I.getNumExplicitDefs() + 2; 1854 i < I.getNumExplicitOperands(); ++i) { 1855 MIB.addImm(I.getOperand(i).getImm()); 1856 } 1857 return MIB.constrainAllUses(TII, TRI, RBI); 1858 } 1859 case Intrinsic::spv_switch: { 1860 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 1861 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 1862 if (I.getOperand(i).isReg()) 1863 MIB.addReg(I.getOperand(i).getReg()); 1864 else if (I.getOperand(i).isCImm()) 1865 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 1866 else if (I.getOperand(i).isMBB()) 1867 MIB.addMBB(I.getOperand(i).getMBB()); 1868 else 1869 llvm_unreachable("Unexpected OpSwitch operand"); 1870 } 1871 return MIB.constrainAllUses(TII, TRI, RBI); 1872 } 1873 case Intrinsic::spv_cmpxchg: 1874 return selectAtomicCmpXchg(ResVReg, ResType, I); 1875 case Intrinsic::spv_unreachable: 1876 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 1877 break; 1878 case Intrinsic::spv_alloca: 1879 return selectFrameIndex(ResVReg, ResType, I); 1880 case Intrinsic::spv_alloca_array: 1881 return selectAllocaArray(ResVReg, ResType, I); 1882 case Intrinsic::spv_assume: 1883 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1884 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 1885 .addUse(I.getOperand(1).getReg()); 1886 break; 1887 case Intrinsic::spv_expect: 1888 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1889 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 1890 .addDef(ResVReg) 1891 .addUse(GR.getSPIRVTypeID(ResType)) 1892 .addUse(I.getOperand(2).getReg()) 1893 .addUse(I.getOperand(3).getReg()); 1894 break; 1895 case Intrinsic::spv_thread_id: 1896 return selectSpvThreadId(ResVReg, ResType, I); 1897 case Intrinsic::spv_all: 1898 return selectAll(ResVReg, ResType, I); 1899 case Intrinsic::spv_any: 1900 return selectAny(ResVReg, ResType, I); 1901 case Intrinsic::spv_lifetime_start: 1902 case Intrinsic::spv_lifetime_end: { 1903 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 1904 : SPIRV::OpLifetimeStop; 1905 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 1906 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 1907 unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg); 1908 bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid; 1909 if (Size == -1 || IsNonvoidPtr) 1910 Size = 0; 1911 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 1912 } break; 1913 default: { 1914 std::string DiagMsg; 1915 raw_string_ostream OS(DiagMsg); 1916 I.print(OS); 1917 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 1918 report_fatal_error(DiagMsg.c_str(), false); 1919 } 1920 } 1921 return true; 1922 } 1923 1924 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 1925 const SPIRVType *ResType, 1926 MachineInstr &I) const { 1927 // there was an allocation size parameter to the allocation instruction 1928 // that is not 1 1929 MachineBasicBlock &BB = *I.getParent(); 1930 return BuildMI(BB, I, I.getDebugLoc(), 1931 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 1932 .addDef(ResVReg) 1933 .addUse(GR.getSPIRVTypeID(ResType)) 1934 .addUse(I.getOperand(2).getReg()) 1935 .constrainAllUses(TII, TRI, RBI); 1936 } 1937 1938 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 1939 const SPIRVType *ResType, 1940 MachineInstr &I) const { 1941 // Change order of instructions if needed: all OpVariable instructions in a 1942 // function must be the first instructions in the first block 1943 MachineFunction *MF = I.getParent()->getParent(); 1944 MachineBasicBlock *MBB = &MF->front(); 1945 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 1946 bool IsHeader = false; 1947 unsigned Opcode; 1948 for (; It != E && It != I; ++It) { 1949 Opcode = It->getOpcode(); 1950 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 1951 IsHeader = true; 1952 } else if (IsHeader && 1953 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 1954 ++It; 1955 break; 1956 } 1957 } 1958 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 1959 .addDef(ResVReg) 1960 .addUse(GR.getSPIRVTypeID(ResType)) 1961 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 1962 .constrainAllUses(TII, TRI, RBI); 1963 } 1964 1965 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 1966 // InstructionSelector walks backwards through the instructions. We can use 1967 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 1968 // first, so can generate an OpBranchConditional here. If there is no 1969 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 1970 const MachineInstr *PrevI = I.getPrevNode(); 1971 MachineBasicBlock &MBB = *I.getParent(); 1972 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 1973 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1974 .addUse(PrevI->getOperand(0).getReg()) 1975 .addMBB(PrevI->getOperand(1).getMBB()) 1976 .addMBB(I.getOperand(0).getMBB()) 1977 .constrainAllUses(TII, TRI, RBI); 1978 } 1979 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 1980 .addMBB(I.getOperand(0).getMBB()) 1981 .constrainAllUses(TII, TRI, RBI); 1982 } 1983 1984 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 1985 // InstructionSelector walks backwards through the instructions. For an 1986 // explicit conditional branch with no fallthrough, we use both a G_BR and a 1987 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 1988 // generate the OpBranchConditional in selectBranch above. 1989 // 1990 // If an OpBranchConditional has been generated, we simply return, as the work 1991 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 1992 // implicit fallthrough to the next basic block, so we need to create an 1993 // OpBranchConditional with an explicit "false" argument pointing to the next 1994 // basic block that LLVM would fall through to. 1995 const MachineInstr *NextI = I.getNextNode(); 1996 // Check if this has already been successfully selected. 1997 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 1998 return true; 1999 // Must be relying on implicit block fallthrough, so generate an 2000 // OpBranchConditional with the "next" basic block as the "false" target. 2001 MachineBasicBlock &MBB = *I.getParent(); 2002 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 2003 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 2004 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 2005 .addUse(I.getOperand(0).getReg()) 2006 .addMBB(I.getOperand(1).getMBB()) 2007 .addMBB(NextMBB) 2008 .constrainAllUses(TII, TRI, RBI); 2009 } 2010 2011 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 2012 const SPIRVType *ResType, 2013 MachineInstr &I) const { 2014 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 2015 .addDef(ResVReg) 2016 .addUse(GR.getSPIRVTypeID(ResType)); 2017 const unsigned NumOps = I.getNumOperands(); 2018 for (unsigned i = 1; i < NumOps; i += 2) { 2019 MIB.addUse(I.getOperand(i + 0).getReg()); 2020 MIB.addMBB(I.getOperand(i + 1).getMBB()); 2021 } 2022 return MIB.constrainAllUses(TII, TRI, RBI); 2023 } 2024 2025 bool SPIRVInstructionSelector::selectGlobalValue( 2026 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 2027 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 2028 MachineIRBuilder MIRBuilder(I); 2029 const GlobalValue *GV = I.getOperand(1).getGlobal(); 2030 Type *GVType = GR.getDeducedGlobalValueType(GV); 2031 SPIRVType *PointerBaseType; 2032 if (GVType->isArrayTy()) { 2033 SPIRVType *ArrayElementType = 2034 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 2035 SPIRV::AccessQualifier::ReadWrite, false); 2036 PointerBaseType = GR.getOrCreateSPIRVArrayType( 2037 ArrayElementType, GVType->getArrayNumElements(), I, TII); 2038 } else { 2039 PointerBaseType = GR.getOrCreateSPIRVType( 2040 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 2041 } 2042 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 2043 PointerBaseType, I, TII, 2044 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 2045 2046 std::string GlobalIdent; 2047 if (!GV->hasName()) { 2048 unsigned &ID = UnnamedGlobalIDs[GV]; 2049 if (ID == 0) 2050 ID = UnnamedGlobalIDs.size(); 2051 GlobalIdent = "__unnamed_" + Twine(ID).str(); 2052 } else { 2053 GlobalIdent = GV->getGlobalIdentifier(); 2054 } 2055 2056 // Behaviour of functions as operands depends on availability of the 2057 // corresponding extension (SPV_INTEL_function_pointers): 2058 // - If there is an extension to operate with functions as operands: 2059 // We create a proper constant operand and evaluate a correct type for a 2060 // function pointer. 2061 // - Without the required extension: 2062 // We have functions as operands in tests with blocks of instruction e.g. in 2063 // transcoding/global_block.ll. These operands are not used and should be 2064 // substituted by zero constants. Their type is expected to be always 2065 // OpTypePointer Function %uchar. 2066 if (isa<Function>(GV)) { 2067 const Constant *ConstVal = GV; 2068 MachineBasicBlock &BB = *I.getParent(); 2069 Register NewReg = GR.find(ConstVal, GR.CurMF); 2070 if (!NewReg.isValid()) { 2071 Register NewReg = ResVReg; 2072 GR.add(ConstVal, GR.CurMF, NewReg); 2073 const Function *GVFun = 2074 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 2075 ? dyn_cast<Function>(GV) 2076 : nullptr; 2077 if (GVFun) { 2078 // References to a function via function pointers generate virtual 2079 // registers without a definition. We will resolve it later, during 2080 // module analysis stage. 2081 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2082 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 2083 MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); 2084 MachineInstrBuilder MB = 2085 BuildMI(BB, I, I.getDebugLoc(), 2086 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 2087 .addDef(NewReg) 2088 .addUse(GR.getSPIRVTypeID(ResType)) 2089 .addUse(FuncVReg); 2090 // mapping the function pointer to the used Function 2091 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 2092 return MB.constrainAllUses(TII, TRI, RBI); 2093 } 2094 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 2095 .addDef(NewReg) 2096 .addUse(GR.getSPIRVTypeID(ResType)) 2097 .constrainAllUses(TII, TRI, RBI); 2098 } 2099 assert(NewReg != ResVReg); 2100 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 2101 .addDef(ResVReg) 2102 .addUse(NewReg) 2103 .constrainAllUses(TII, TRI, RBI); 2104 } 2105 auto GlobalVar = cast<GlobalVariable>(GV); 2106 assert(GlobalVar->getName() != "llvm.global.annotations"); 2107 2108 bool HasInit = GlobalVar->hasInitializer() && 2109 !isa<UndefValue>(GlobalVar->getInitializer()); 2110 // Skip empty declaration for GVs with initilaizers till we get the decl with 2111 // passed initializer. 2112 if (HasInit && !Init) 2113 return true; 2114 2115 unsigned AddrSpace = GV->getAddressSpace(); 2116 SPIRV::StorageClass::StorageClass Storage = 2117 addressSpaceToStorageClass(AddrSpace, STI); 2118 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 2119 Storage != SPIRV::StorageClass::Function; 2120 SPIRV::LinkageType::LinkageType LnkType = 2121 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 2122 ? SPIRV::LinkageType::Import 2123 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 2124 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 2125 ? SPIRV::LinkageType::LinkOnceODR 2126 : SPIRV::LinkageType::Export); 2127 2128 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 2129 Storage, Init, GlobalVar->isConstant(), 2130 HasLnkTy, LnkType, MIRBuilder, true); 2131 return Reg.isValid(); 2132 } 2133 2134 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 2135 const SPIRVType *ResType, 2136 MachineInstr &I) const { 2137 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 2138 return selectExtInst(ResVReg, ResType, I, CL::log10); 2139 } 2140 2141 // There is no log10 instruction in the GLSL Extended Instruction set, so it 2142 // is implemented as: 2143 // log10(x) = log2(x) * (1 / log2(10)) 2144 // = log2(x) * 0.30103 2145 2146 MachineIRBuilder MIRBuilder(I); 2147 MachineBasicBlock &BB = *I.getParent(); 2148 2149 // Build log2(x). 2150 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2151 bool Result = 2152 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2153 .addDef(VarReg) 2154 .addUse(GR.getSPIRVTypeID(ResType)) 2155 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2156 .addImm(GL::Log2) 2157 .add(I.getOperand(1)) 2158 .constrainAllUses(TII, TRI, RBI); 2159 2160 // Build 0.30103. 2161 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 2162 ResType->getOpcode() == SPIRV::OpTypeFloat); 2163 // TODO: Add matrix implementation once supported by the HLSL frontend. 2164 const SPIRVType *SpirvScalarType = 2165 ResType->getOpcode() == SPIRV::OpTypeVector 2166 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 2167 : ResType; 2168 Register ScaleReg = 2169 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 2170 2171 // Multiply log2(x) by 0.30103 to get log10(x) result. 2172 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 2173 ? SPIRV::OpVectorTimesScalar 2174 : SPIRV::OpFMulS; 2175 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2176 .addDef(ResVReg) 2177 .addUse(GR.getSPIRVTypeID(ResType)) 2178 .addUse(VarReg) 2179 .addUse(ScaleReg) 2180 .constrainAllUses(TII, TRI, RBI); 2181 2182 return Result; 2183 } 2184 2185 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 2186 const SPIRVType *ResType, 2187 MachineInstr &I) const { 2188 // DX intrinsic: @llvm.dx.thread.id(i32) 2189 // ID Name Description 2190 // 93 ThreadId reads the thread ID 2191 2192 MachineIRBuilder MIRBuilder(I); 2193 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 2194 const SPIRVType *Vec3Ty = 2195 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 2196 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 2197 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 2198 2199 // Create new register for GlobalInvocationID builtin variable. 2200 Register NewRegister = 2201 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); 2202 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32)); 2203 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 2204 2205 // Build GlobalInvocationID global variable with the necessary decorations. 2206 Register Variable = GR.buildGlobalVariable( 2207 NewRegister, PtrType, 2208 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 2209 SPIRV::StorageClass::Input, nullptr, true, true, 2210 SPIRV::LinkageType::Import, MIRBuilder, false); 2211 2212 // Create new register for loading value. 2213 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2214 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2215 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32)); 2216 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 2217 2218 // Load v3uint value from the global variable. 2219 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2220 .addDef(LoadedRegister) 2221 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 2222 .addUse(Variable); 2223 2224 // Get Thread ID index. Expecting operand is a constant immediate value, 2225 // wrapped in a type assignment. 2226 assert(I.getOperand(2).isReg()); 2227 Register ThreadIdReg = I.getOperand(2).getReg(); 2228 SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg); 2229 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 2230 ConstTy->getOperand(1).isReg()); 2231 Register ConstReg = ConstTy->getOperand(1).getReg(); 2232 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 2233 assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); 2234 const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue(); 2235 const uint32_t ThreadId = Val.getZExtValue(); 2236 2237 // Extract the thread ID from the loaded vector value. 2238 MachineBasicBlock &BB = *I.getParent(); 2239 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2240 .addDef(ResVReg) 2241 .addUse(GR.getSPIRVTypeID(ResType)) 2242 .addUse(LoadedRegister) 2243 .addImm(ThreadId); 2244 return MIB.constrainAllUses(TII, TRI, RBI); 2245 } 2246 2247 namespace llvm { 2248 InstructionSelector * 2249 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 2250 const SPIRVSubtarget &Subtarget, 2251 const RegisterBankInfo &RBI) { 2252 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 2253 } 2254 } // namespace llvm 2255