1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 16 #include "SPIRV.h" 17 #include "SPIRVGlobalRegistry.h" 18 #include "SPIRVInstrInfo.h" 19 #include "SPIRVRegisterBankInfo.h" 20 #include "SPIRVRegisterInfo.h" 21 #include "SPIRVTargetMachine.h" 22 #include "SPIRVUtils.h" 23 #include "llvm/ADT/APFloat.h" 24 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 25 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 26 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 27 #include "llvm/CodeGen/MachineInstrBuilder.h" 28 #include "llvm/CodeGen/MachineRegisterInfo.h" 29 #include "llvm/IR/IntrinsicsSPIRV.h" 30 #include "llvm/Support/Debug.h" 31 32 #define DEBUG_TYPE "spirv-isel" 33 34 using namespace llvm; 35 namespace CL = SPIRV::OpenCLExtInst; 36 namespace GL = SPIRV::GLSLExtInst; 37 38 using ExtInstList = 39 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 40 41 namespace { 42 43 #define GET_GLOBALISEL_PREDICATE_BITSET 44 #include "SPIRVGenGlobalISel.inc" 45 #undef GET_GLOBALISEL_PREDICATE_BITSET 46 47 class SPIRVInstructionSelector : public InstructionSelector { 48 const SPIRVSubtarget &STI; 49 const SPIRVInstrInfo &TII; 50 const SPIRVRegisterInfo &TRI; 51 const RegisterBankInfo &RBI; 52 SPIRVGlobalRegistry &GR; 53 MachineRegisterInfo *MRI; 54 55 public: 56 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 57 const SPIRVSubtarget &ST, 58 const RegisterBankInfo &RBI); 59 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 60 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 61 BlockFrequencyInfo *BFI) override; 62 // Common selection code. Instruction-specific selection occurs in spvSelect. 63 bool select(MachineInstr &I) override; 64 static const char *getName() { return DEBUG_TYPE; } 65 66 #define GET_GLOBALISEL_PREDICATES_DECL 67 #include "SPIRVGenGlobalISel.inc" 68 #undef GET_GLOBALISEL_PREDICATES_DECL 69 70 #define GET_GLOBALISEL_TEMPORARIES_DECL 71 #include "SPIRVGenGlobalISel.inc" 72 #undef GET_GLOBALISEL_TEMPORARIES_DECL 73 74 private: 75 // tblgen-erated 'select' implementation, used as the initial selector for 76 // the patterns that don't require complex C++. 77 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 78 79 // All instruction-specific selection that didn't happen in "select()". 80 // Is basically a large Switch/Case delegating to all other select method. 81 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 82 MachineInstr &I) const; 83 84 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 85 const MachineInstr *Init = nullptr) const; 86 87 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 88 MachineInstr &I, Register SrcReg, 89 unsigned Opcode) const; 90 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 91 unsigned Opcode) const; 92 93 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 94 MachineInstr &I) const; 95 bool selectStore(MachineInstr &I) const; 96 97 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 98 99 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 100 MachineInstr &I, unsigned NewOpcode) const; 101 102 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 103 MachineInstr &I) const; 104 105 bool selectFence(MachineInstr &I) const; 106 107 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 108 MachineInstr &I) const; 109 110 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 111 MachineInstr &I) const; 112 113 bool selectConstVector(Register ResVReg, const SPIRVType *ResType, 114 MachineInstr &I) const; 115 116 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 117 unsigned comparisonOpcode, MachineInstr &I) const; 118 119 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I) const; 121 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 122 MachineInstr &I) const; 123 124 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 125 int OpIdx) const; 126 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 127 int OpIdx) const; 128 129 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 130 MachineInstr &I) const; 131 132 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 133 bool IsSigned) const; 134 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 135 bool IsSigned, unsigned Opcode) const; 136 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 137 bool IsSigned) const; 138 139 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 140 MachineInstr &I) const; 141 142 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 143 const SPIRVType *intTy, const SPIRVType *boolTy) const; 144 145 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 148 MachineInstr &I) const; 149 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 150 MachineInstr &I) const; 151 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 152 MachineInstr &I) const; 153 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 154 MachineInstr &I) const; 155 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 158 MachineInstr &I) const; 159 160 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 163 bool selectBranch(MachineInstr &I) const; 164 bool selectBranchCond(MachineInstr &I) const; 165 166 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 167 MachineInstr &I) const; 168 169 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 170 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 171 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 172 MachineInstr &I, CL::OpenCLExtInst CLInst, 173 GL::GLSLExtInst GLInst) const; 174 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 175 MachineInstr &I, const ExtInstList &ExtInsts) const; 176 177 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 178 MachineInstr &I) const; 179 180 Register buildI32Constant(uint32_t Val, MachineInstr &I, 181 const SPIRVType *ResType = nullptr) const; 182 183 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 184 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 185 MachineInstr &I) const; 186 }; 187 188 } // end anonymous namespace 189 190 #define GET_GLOBALISEL_IMPL 191 #include "SPIRVGenGlobalISel.inc" 192 #undef GET_GLOBALISEL_IMPL 193 194 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 195 const SPIRVSubtarget &ST, 196 const RegisterBankInfo &RBI) 197 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 198 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 199 #define GET_GLOBALISEL_PREDICATES_INIT 200 #include "SPIRVGenGlobalISel.inc" 201 #undef GET_GLOBALISEL_PREDICATES_INIT 202 #define GET_GLOBALISEL_TEMPORARIES_INIT 203 #include "SPIRVGenGlobalISel.inc" 204 #undef GET_GLOBALISEL_TEMPORARIES_INIT 205 { 206 } 207 208 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 209 CodeGenCoverage *CoverageInfo, 210 ProfileSummaryInfo *PSI, 211 BlockFrequencyInfo *BFI) { 212 MRI = &MF.getRegInfo(); 213 GR.setCurrentFunc(MF); 214 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 215 } 216 217 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 218 219 // Defined in SPIRVLegalizerInfo.cpp. 220 extern bool isTypeFoldingSupported(unsigned Opcode); 221 222 bool SPIRVInstructionSelector::select(MachineInstr &I) { 223 assert(I.getParent() && "Instruction should be in a basic block!"); 224 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 225 226 Register Opcode = I.getOpcode(); 227 // If it's not a GMIR instruction, we've selected it already. 228 if (!isPreISelGenericOpcode(Opcode)) { 229 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 230 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); 231 if (isTypeFoldingSupported(Def->getOpcode())) { 232 auto Res = selectImpl(I, *CoverageInfo); 233 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 234 if (Res) 235 return Res; 236 } 237 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); 238 I.removeFromParent(); 239 return true; 240 } else if (I.getNumDefs() == 1) { 241 // Make all vregs 32 bits (for SPIR-V IDs). 242 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); 243 } 244 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 245 } 246 247 if (I.getNumOperands() != I.getNumExplicitOperands()) { 248 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 249 return false; 250 } 251 252 // Common code for getting return reg+type, and removing selected instr 253 // from parent occurs here. Instr-specific selection happens in spvSelect(). 254 bool HasDefs = I.getNumDefs() > 0; 255 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 256 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 257 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 258 if (spvSelect(ResVReg, ResType, I)) { 259 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). 260 MRI->setType(ResVReg, LLT::scalar(32)); 261 I.removeFromParent(); 262 return true; 263 } 264 return false; 265 } 266 267 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 268 const SPIRVType *ResType, 269 MachineInstr &I) const { 270 assert(!isTypeFoldingSupported(I.getOpcode()) || 271 I.getOpcode() == TargetOpcode::G_CONSTANT); 272 const unsigned Opcode = I.getOpcode(); 273 switch (Opcode) { 274 case TargetOpcode::G_CONSTANT: 275 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 276 I); 277 case TargetOpcode::G_GLOBAL_VALUE: 278 return selectGlobalValue(ResVReg, I); 279 case TargetOpcode::G_IMPLICIT_DEF: 280 return selectOpUndef(ResVReg, ResType, I); 281 282 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 283 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 284 return selectIntrinsic(ResVReg, ResType, I); 285 case TargetOpcode::G_BITREVERSE: 286 return selectBitreverse(ResVReg, ResType, I); 287 288 case TargetOpcode::G_BUILD_VECTOR: 289 return selectConstVector(ResVReg, ResType, I); 290 291 case TargetOpcode::G_SHUFFLE_VECTOR: { 292 MachineBasicBlock &BB = *I.getParent(); 293 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 294 .addDef(ResVReg) 295 .addUse(GR.getSPIRVTypeID(ResType)) 296 .addUse(I.getOperand(1).getReg()) 297 .addUse(I.getOperand(2).getReg()); 298 for (auto V : I.getOperand(3).getShuffleMask()) 299 MIB.addImm(V); 300 return MIB.constrainAllUses(TII, TRI, RBI); 301 } 302 case TargetOpcode::G_MEMMOVE: 303 case TargetOpcode::G_MEMCPY: 304 case TargetOpcode::G_MEMSET: 305 return selectMemOperation(ResVReg, I); 306 307 case TargetOpcode::G_ICMP: 308 return selectICmp(ResVReg, ResType, I); 309 case TargetOpcode::G_FCMP: 310 return selectFCmp(ResVReg, ResType, I); 311 312 case TargetOpcode::G_FRAME_INDEX: 313 return selectFrameIndex(ResVReg, ResType, I); 314 315 case TargetOpcode::G_LOAD: 316 return selectLoad(ResVReg, ResType, I); 317 case TargetOpcode::G_STORE: 318 return selectStore(I); 319 320 case TargetOpcode::G_BR: 321 return selectBranch(I); 322 case TargetOpcode::G_BRCOND: 323 return selectBranchCond(I); 324 325 case TargetOpcode::G_PHI: 326 return selectPhi(ResVReg, ResType, I); 327 328 case TargetOpcode::G_FPTOSI: 329 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 330 case TargetOpcode::G_FPTOUI: 331 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 332 333 case TargetOpcode::G_SITOFP: 334 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 335 case TargetOpcode::G_UITOFP: 336 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 337 338 case TargetOpcode::G_CTPOP: 339 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 340 case TargetOpcode::G_SMIN: 341 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 342 case TargetOpcode::G_UMIN: 343 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 344 345 case TargetOpcode::G_SMAX: 346 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 347 case TargetOpcode::G_UMAX: 348 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 349 350 case TargetOpcode::G_FMA: 351 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 352 353 case TargetOpcode::G_FPOW: 354 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 355 case TargetOpcode::G_FPOWI: 356 return selectExtInst(ResVReg, ResType, I, CL::pown); 357 358 case TargetOpcode::G_FEXP: 359 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 360 case TargetOpcode::G_FEXP2: 361 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 362 363 case TargetOpcode::G_FLOG: 364 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 365 case TargetOpcode::G_FLOG2: 366 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 367 case TargetOpcode::G_FLOG10: 368 return selectLog10(ResVReg, ResType, I); 369 370 case TargetOpcode::G_FABS: 371 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 372 case TargetOpcode::G_ABS: 373 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 374 375 case TargetOpcode::G_FMINNUM: 376 case TargetOpcode::G_FMINIMUM: 377 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin); 378 case TargetOpcode::G_FMAXNUM: 379 case TargetOpcode::G_FMAXIMUM: 380 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax); 381 382 case TargetOpcode::G_FCOPYSIGN: 383 return selectExtInst(ResVReg, ResType, I, CL::copysign); 384 385 case TargetOpcode::G_FCEIL: 386 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 387 case TargetOpcode::G_FFLOOR: 388 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 389 390 case TargetOpcode::G_FCOS: 391 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 392 case TargetOpcode::G_FSIN: 393 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 394 395 case TargetOpcode::G_FSQRT: 396 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 397 398 case TargetOpcode::G_CTTZ: 399 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 400 return selectExtInst(ResVReg, ResType, I, CL::ctz); 401 case TargetOpcode::G_CTLZ: 402 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 403 return selectExtInst(ResVReg, ResType, I, CL::clz); 404 405 case TargetOpcode::G_INTRINSIC_ROUND: 406 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 407 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 408 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 409 case TargetOpcode::G_INTRINSIC_TRUNC: 410 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 411 case TargetOpcode::G_FRINT: 412 case TargetOpcode::G_FNEARBYINT: 413 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 414 415 case TargetOpcode::G_SMULH: 416 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 417 case TargetOpcode::G_UMULH: 418 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 419 420 case TargetOpcode::G_SEXT: 421 return selectExt(ResVReg, ResType, I, true); 422 case TargetOpcode::G_ANYEXT: 423 case TargetOpcode::G_ZEXT: 424 return selectExt(ResVReg, ResType, I, false); 425 case TargetOpcode::G_TRUNC: 426 return selectTrunc(ResVReg, ResType, I); 427 case TargetOpcode::G_FPTRUNC: 428 case TargetOpcode::G_FPEXT: 429 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 430 431 case TargetOpcode::G_PTRTOINT: 432 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 433 case TargetOpcode::G_INTTOPTR: 434 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 435 case TargetOpcode::G_BITCAST: 436 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 437 case TargetOpcode::G_ADDRSPACE_CAST: 438 return selectAddrSpaceCast(ResVReg, ResType, I); 439 case TargetOpcode::G_PTR_ADD: { 440 // Currently, we get G_PTR_ADD only as a result of translating 441 // global variables, initialized with constant expressions like GV + Const 442 // (see test opencl/basic/progvar_prog_scope_init.ll). 443 // TODO: extend the handler once we have other cases. 444 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 445 Register GV = I.getOperand(1).getReg(); 446 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 447 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 448 (*II).getOpcode() == TargetOpcode::COPY || 449 (*II).getOpcode() == SPIRV::OpVariable) && 450 isImm(I.getOperand(2), MRI)); 451 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 452 MachineBasicBlock &BB = *I.getParent(); 453 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 454 .addDef(ResVReg) 455 .addUse(GR.getSPIRVTypeID(ResType)) 456 .addImm(static_cast<uint32_t>( 457 SPIRV::Opcode::InBoundsPtrAccessChain)) 458 .addUse(GV) 459 .addUse(Idx) 460 .addUse(I.getOperand(2).getReg()); 461 return MIB.constrainAllUses(TII, TRI, RBI); 462 } 463 464 case TargetOpcode::G_ATOMICRMW_OR: 465 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 466 case TargetOpcode::G_ATOMICRMW_ADD: 467 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 468 case TargetOpcode::G_ATOMICRMW_AND: 469 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 470 case TargetOpcode::G_ATOMICRMW_MAX: 471 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 472 case TargetOpcode::G_ATOMICRMW_MIN: 473 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 474 case TargetOpcode::G_ATOMICRMW_SUB: 475 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 476 case TargetOpcode::G_ATOMICRMW_XOR: 477 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 478 case TargetOpcode::G_ATOMICRMW_UMAX: 479 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 480 case TargetOpcode::G_ATOMICRMW_UMIN: 481 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 482 case TargetOpcode::G_ATOMICRMW_XCHG: 483 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 484 case TargetOpcode::G_ATOMIC_CMPXCHG: 485 return selectAtomicCmpXchg(ResVReg, ResType, I); 486 487 case TargetOpcode::G_FENCE: 488 return selectFence(I); 489 490 default: 491 return false; 492 } 493 } 494 495 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 496 const SPIRVType *ResType, 497 MachineInstr &I, 498 CL::OpenCLExtInst CLInst) const { 499 return selectExtInst(ResVReg, ResType, I, 500 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 501 } 502 503 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 504 const SPIRVType *ResType, 505 MachineInstr &I, 506 CL::OpenCLExtInst CLInst, 507 GL::GLSLExtInst GLInst) const { 508 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 509 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 510 return selectExtInst(ResVReg, ResType, I, ExtInsts); 511 } 512 513 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 514 const SPIRVType *ResType, 515 MachineInstr &I, 516 const ExtInstList &Insts) const { 517 518 for (const auto &Ex : Insts) { 519 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 520 uint32_t Opcode = Ex.second; 521 if (STI.canUseExtInstSet(Set)) { 522 MachineBasicBlock &BB = *I.getParent(); 523 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 524 .addDef(ResVReg) 525 .addUse(GR.getSPIRVTypeID(ResType)) 526 .addImm(static_cast<uint32_t>(Set)) 527 .addImm(Opcode); 528 const unsigned NumOps = I.getNumOperands(); 529 for (unsigned i = 1; i < NumOps; ++i) 530 MIB.add(I.getOperand(i)); 531 return MIB.constrainAllUses(TII, TRI, RBI); 532 } 533 } 534 return false; 535 } 536 537 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 538 const SPIRVType *ResType, 539 MachineInstr &I, 540 Register SrcReg, 541 unsigned Opcode) const { 542 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 543 .addDef(ResVReg) 544 .addUse(GR.getSPIRVTypeID(ResType)) 545 .addUse(SrcReg) 546 .constrainAllUses(TII, TRI, RBI); 547 } 548 549 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 550 const SPIRVType *ResType, 551 MachineInstr &I, 552 unsigned Opcode) const { 553 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 554 Opcode); 555 } 556 557 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) { 558 switch (Ord) { 559 case SyncScope::SingleThread: 560 return SPIRV::Scope::Invocation; 561 case SyncScope::System: 562 return SPIRV::Scope::Device; 563 default: 564 llvm_unreachable("Unsupported synchronization Scope ID."); 565 } 566 } 567 568 static void addMemoryOperands(MachineMemOperand *MemOp, 569 MachineInstrBuilder &MIB) { 570 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 571 if (MemOp->isVolatile()) 572 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 573 if (MemOp->isNonTemporal()) 574 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 575 if (MemOp->getAlign().value()) 576 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 577 578 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 579 MIB.addImm(SpvMemOp); 580 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 581 MIB.addImm(MemOp->getAlign().value()); 582 } 583 } 584 585 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 586 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 587 if (Flags & MachineMemOperand::Flags::MOVolatile) 588 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 589 if (Flags & MachineMemOperand::Flags::MONonTemporal) 590 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 591 592 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 593 MIB.addImm(SpvMemOp); 594 } 595 596 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 597 const SPIRVType *ResType, 598 MachineInstr &I) const { 599 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 600 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 601 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 602 .addDef(ResVReg) 603 .addUse(GR.getSPIRVTypeID(ResType)) 604 .addUse(Ptr); 605 if (!I.getNumMemOperands()) { 606 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 607 I.getOpcode() == 608 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 609 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 610 } else { 611 addMemoryOperands(*I.memoperands_begin(), MIB); 612 } 613 return MIB.constrainAllUses(TII, TRI, RBI); 614 } 615 616 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 617 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 618 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 619 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 620 MachineBasicBlock &BB = *I.getParent(); 621 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 622 .addUse(Ptr) 623 .addUse(StoreVal); 624 if (!I.getNumMemOperands()) { 625 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 626 I.getOpcode() == 627 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 628 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 629 } else { 630 addMemoryOperands(*I.memoperands_begin(), MIB); 631 } 632 return MIB.constrainAllUses(TII, TRI, RBI); 633 } 634 635 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 636 MachineInstr &I) const { 637 MachineBasicBlock &BB = *I.getParent(); 638 Register SrcReg = I.getOperand(1).getReg(); 639 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 640 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 641 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 642 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 643 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 644 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 645 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); 646 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 647 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 648 // TODO: check if we have such GV, add init, use buildGlobalVariable. 649 Type *LLVMArrTy = ArrayType::get( 650 IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); 651 GlobalVariable *GV = 652 new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); 653 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 654 GR.add(GV, GR.CurMF, VarReg); 655 656 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 657 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 658 .addDef(VarReg) 659 .addUse(GR.getSPIRVTypeID(VarTy)) 660 .addImm(SPIRV::StorageClass::UniformConstant) 661 .addUse(Const) 662 .constrainAllUses(TII, TRI, RBI); 663 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 664 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 665 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 666 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 667 } 668 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 669 .addUse(I.getOperand(0).getReg()) 670 .addUse(SrcReg) 671 .addUse(I.getOperand(2).getReg()); 672 if (I.getNumMemOperands()) 673 addMemoryOperands(*I.memoperands_begin(), MIB); 674 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 675 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 676 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 677 .addUse(MIB->getOperand(0).getReg()); 678 return Result; 679 } 680 681 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 682 const SPIRVType *ResType, 683 MachineInstr &I, 684 unsigned NewOpcode) const { 685 assert(I.hasOneMemOperand()); 686 const MachineMemOperand *MemOp = *I.memoperands_begin(); 687 uint32_t Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID())); 688 Register ScopeReg = buildI32Constant(Scope, I); 689 690 Register Ptr = I.getOperand(1).getReg(); 691 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 692 // auto ScSem = 693 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 694 AtomicOrdering AO = MemOp->getSuccessOrdering(); 695 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 696 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 697 698 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 699 .addDef(ResVReg) 700 .addUse(GR.getSPIRVTypeID(ResType)) 701 .addUse(Ptr) 702 .addUse(ScopeReg) 703 .addUse(MemSemReg) 704 .addUse(I.getOperand(2).getReg()) 705 .constrainAllUses(TII, TRI, RBI); 706 } 707 708 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 709 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 710 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 711 Register MemSemReg = buildI32Constant(MemSem, I); 712 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 713 uint32_t Scope = static_cast<uint32_t>(getScope(Ord)); 714 Register ScopeReg = buildI32Constant(Scope, I); 715 MachineBasicBlock &BB = *I.getParent(); 716 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 717 .addUse(ScopeReg) 718 .addUse(MemSemReg) 719 .constrainAllUses(TII, TRI, RBI); 720 } 721 722 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 723 const SPIRVType *ResType, 724 MachineInstr &I) const { 725 Register ScopeReg; 726 Register MemSemEqReg; 727 Register MemSemNeqReg; 728 Register Ptr = I.getOperand(2).getReg(); 729 if (!isa<GIntrinsic>(I)) { 730 assert(I.hasOneMemOperand()); 731 const MachineMemOperand *MemOp = *I.memoperands_begin(); 732 unsigned Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID())); 733 ScopeReg = buildI32Constant(Scope, I); 734 735 unsigned ScSem = static_cast<uint32_t>( 736 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 737 AtomicOrdering AO = MemOp->getSuccessOrdering(); 738 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 739 MemSemEqReg = buildI32Constant(MemSemEq, I); 740 AtomicOrdering FO = MemOp->getFailureOrdering(); 741 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 742 MemSemNeqReg = 743 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 744 } else { 745 ScopeReg = I.getOperand(5).getReg(); 746 MemSemEqReg = I.getOperand(6).getReg(); 747 MemSemNeqReg = I.getOperand(7).getReg(); 748 } 749 750 Register Cmp = I.getOperand(3).getReg(); 751 Register Val = I.getOperand(4).getReg(); 752 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 753 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); 754 const DebugLoc &DL = I.getDebugLoc(); 755 bool Result = 756 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 757 .addDef(ACmpRes) 758 .addUse(GR.getSPIRVTypeID(SpvValTy)) 759 .addUse(Ptr) 760 .addUse(ScopeReg) 761 .addUse(MemSemEqReg) 762 .addUse(MemSemNeqReg) 763 .addUse(Val) 764 .addUse(Cmp) 765 .constrainAllUses(TII, TRI, RBI); 766 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 767 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 768 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 769 .addDef(CmpSuccReg) 770 .addUse(GR.getSPIRVTypeID(BoolTy)) 771 .addUse(ACmpRes) 772 .addUse(Cmp) 773 .constrainAllUses(TII, TRI, RBI); 774 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 775 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 776 .addDef(TmpReg) 777 .addUse(GR.getSPIRVTypeID(ResType)) 778 .addUse(ACmpRes) 779 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 780 .addImm(0) 781 .constrainAllUses(TII, TRI, RBI); 782 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 783 .addDef(ResVReg) 784 .addUse(GR.getSPIRVTypeID(ResType)) 785 .addUse(CmpSuccReg) 786 .addUse(TmpReg) 787 .addImm(1) 788 .constrainAllUses(TII, TRI, RBI); 789 return Result; 790 } 791 792 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 793 switch (SC) { 794 case SPIRV::StorageClass::Workgroup: 795 case SPIRV::StorageClass::CrossWorkgroup: 796 case SPIRV::StorageClass::Function: 797 return true; 798 default: 799 return false; 800 } 801 } 802 803 // In SPIR-V address space casting can only happen to and from the Generic 804 // storage class. We can also only case Workgroup, CrossWorkgroup, or Function 805 // pointers to and from Generic pointers. As such, we can convert e.g. from 806 // Workgroup to Function by going via a Generic pointer as an intermediary. All 807 // other combinations can only be done by a bitcast, and are probably not safe. 808 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 809 const SPIRVType *ResType, 810 MachineInstr &I) const { 811 // If the AddrSpaceCast user is single and in OpConstantComposite or 812 // OpVariable, we should select OpSpecConstantOp. 813 auto UIs = MRI->use_instructions(ResVReg); 814 if (!UIs.empty() && ++UIs.begin() == UIs.end() && 815 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || 816 UIs.begin()->getOpcode() == SPIRV::OpVariable || 817 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { 818 Register NewReg = I.getOperand(1).getReg(); 819 MachineBasicBlock &BB = *I.getParent(); 820 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 821 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, 822 SPIRV::StorageClass::Generic); 823 bool Result = 824 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 825 .addDef(ResVReg) 826 .addUse(GR.getSPIRVTypeID(ResType)) 827 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) 828 .addUse(NewReg) 829 .constrainAllUses(TII, TRI, RBI); 830 return Result; 831 } 832 Register SrcPtr = I.getOperand(1).getReg(); 833 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 834 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); 835 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); 836 837 // Casting from an eligable pointer to Generic. 838 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 839 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 840 // Casting from Generic to an eligable pointer. 841 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 842 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 843 // Casting between 2 eligable pointers using Generic as an intermediary. 844 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 845 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); 846 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 847 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); 848 MachineBasicBlock &BB = *I.getParent(); 849 const DebugLoc &DL = I.getDebugLoc(); 850 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 851 .addDef(Tmp) 852 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 853 .addUse(SrcPtr) 854 .constrainAllUses(TII, TRI, RBI); 855 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 856 .addDef(ResVReg) 857 .addUse(GR.getSPIRVTypeID(ResType)) 858 .addUse(Tmp) 859 .constrainAllUses(TII, TRI, RBI); 860 } 861 // TODO Should this case just be disallowed completely? 862 // We're casting 2 other arbitrary address spaces, so have to bitcast. 863 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 864 } 865 866 static unsigned getFCmpOpcode(unsigned PredNum) { 867 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 868 switch (Pred) { 869 case CmpInst::FCMP_OEQ: 870 return SPIRV::OpFOrdEqual; 871 case CmpInst::FCMP_OGE: 872 return SPIRV::OpFOrdGreaterThanEqual; 873 case CmpInst::FCMP_OGT: 874 return SPIRV::OpFOrdGreaterThan; 875 case CmpInst::FCMP_OLE: 876 return SPIRV::OpFOrdLessThanEqual; 877 case CmpInst::FCMP_OLT: 878 return SPIRV::OpFOrdLessThan; 879 case CmpInst::FCMP_ONE: 880 return SPIRV::OpFOrdNotEqual; 881 case CmpInst::FCMP_ORD: 882 return SPIRV::OpOrdered; 883 case CmpInst::FCMP_UEQ: 884 return SPIRV::OpFUnordEqual; 885 case CmpInst::FCMP_UGE: 886 return SPIRV::OpFUnordGreaterThanEqual; 887 case CmpInst::FCMP_UGT: 888 return SPIRV::OpFUnordGreaterThan; 889 case CmpInst::FCMP_ULE: 890 return SPIRV::OpFUnordLessThanEqual; 891 case CmpInst::FCMP_ULT: 892 return SPIRV::OpFUnordLessThan; 893 case CmpInst::FCMP_UNE: 894 return SPIRV::OpFUnordNotEqual; 895 case CmpInst::FCMP_UNO: 896 return SPIRV::OpUnordered; 897 default: 898 llvm_unreachable("Unknown predicate type for FCmp"); 899 } 900 } 901 902 static unsigned getICmpOpcode(unsigned PredNum) { 903 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 904 switch (Pred) { 905 case CmpInst::ICMP_EQ: 906 return SPIRV::OpIEqual; 907 case CmpInst::ICMP_NE: 908 return SPIRV::OpINotEqual; 909 case CmpInst::ICMP_SGE: 910 return SPIRV::OpSGreaterThanEqual; 911 case CmpInst::ICMP_SGT: 912 return SPIRV::OpSGreaterThan; 913 case CmpInst::ICMP_SLE: 914 return SPIRV::OpSLessThanEqual; 915 case CmpInst::ICMP_SLT: 916 return SPIRV::OpSLessThan; 917 case CmpInst::ICMP_UGE: 918 return SPIRV::OpUGreaterThanEqual; 919 case CmpInst::ICMP_UGT: 920 return SPIRV::OpUGreaterThan; 921 case CmpInst::ICMP_ULE: 922 return SPIRV::OpULessThanEqual; 923 case CmpInst::ICMP_ULT: 924 return SPIRV::OpULessThan; 925 default: 926 llvm_unreachable("Unknown predicate type for ICmp"); 927 } 928 } 929 930 static unsigned getPtrCmpOpcode(unsigned Pred) { 931 switch (static_cast<CmpInst::Predicate>(Pred)) { 932 case CmpInst::ICMP_EQ: 933 return SPIRV::OpPtrEqual; 934 case CmpInst::ICMP_NE: 935 return SPIRV::OpPtrNotEqual; 936 default: 937 llvm_unreachable("Unknown predicate type for pointer comparison"); 938 } 939 } 940 941 // Return the logical operation, or abort if none exists. 942 static unsigned getBoolCmpOpcode(unsigned PredNum) { 943 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 944 switch (Pred) { 945 case CmpInst::ICMP_EQ: 946 return SPIRV::OpLogicalEqual; 947 case CmpInst::ICMP_NE: 948 return SPIRV::OpLogicalNotEqual; 949 default: 950 llvm_unreachable("Unknown predicate type for Bool comparison"); 951 } 952 } 953 954 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 955 const SPIRVType *ResType, 956 MachineInstr &I) const { 957 MachineBasicBlock &BB = *I.getParent(); 958 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 959 .addDef(ResVReg) 960 .addUse(GR.getSPIRVTypeID(ResType)) 961 .addUse(I.getOperand(1).getReg()) 962 .constrainAllUses(TII, TRI, RBI); 963 } 964 965 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, 966 const SPIRVType *ResType, 967 MachineInstr &I) const { 968 // TODO: only const case is supported for now. 969 assert(std::all_of( 970 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { 971 if (MO.isDef()) 972 return true; 973 if (!MO.isReg()) 974 return false; 975 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); 976 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 977 ConstTy->getOperand(1).isReg()); 978 Register ConstReg = ConstTy->getOperand(1).getReg(); 979 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 980 assert(Const); 981 return (Const->getOpcode() == TargetOpcode::G_CONSTANT || 982 Const->getOpcode() == TargetOpcode::G_FCONSTANT); 983 })); 984 985 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 986 TII.get(SPIRV::OpConstantComposite)) 987 .addDef(ResVReg) 988 .addUse(GR.getSPIRVTypeID(ResType)); 989 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 990 MIB.addUse(I.getOperand(i).getReg()); 991 return MIB.constrainAllUses(TII, TRI, RBI); 992 } 993 994 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 995 const SPIRVType *ResType, 996 unsigned CmpOpc, 997 MachineInstr &I) const { 998 Register Cmp0 = I.getOperand(2).getReg(); 999 Register Cmp1 = I.getOperand(3).getReg(); 1000 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1001 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1002 "CMP operands should have the same type"); 1003 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1004 .addDef(ResVReg) 1005 .addUse(GR.getSPIRVTypeID(ResType)) 1006 .addUse(Cmp0) 1007 .addUse(Cmp1) 1008 .constrainAllUses(TII, TRI, RBI); 1009 } 1010 1011 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1012 const SPIRVType *ResType, 1013 MachineInstr &I) const { 1014 auto Pred = I.getOperand(1).getPredicate(); 1015 unsigned CmpOpc; 1016 1017 Register CmpOperand = I.getOperand(2).getReg(); 1018 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1019 CmpOpc = getPtrCmpOpcode(Pred); 1020 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1021 CmpOpc = getBoolCmpOpcode(Pred); 1022 else 1023 CmpOpc = getICmpOpcode(Pred); 1024 return selectCmp(ResVReg, ResType, CmpOpc, I); 1025 } 1026 1027 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, 1028 const MachineInstr &I, 1029 int OpIdx) const { 1030 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1031 "Expected G_FCONSTANT"); 1032 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1033 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1034 } 1035 1036 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1037 const MachineInstr &I, 1038 int OpIdx) const { 1039 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1040 "Expected G_CONSTANT"); 1041 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1042 } 1043 1044 Register 1045 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1046 const SPIRVType *ResType) const { 1047 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1048 const SPIRVType *SpvI32Ty = 1049 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1050 // Find a constant in DT or build a new one. 1051 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1052 Register NewReg = GR.find(ConstInt, GR.CurMF); 1053 if (!NewReg.isValid()) { 1054 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1055 GR.add(ConstInt, GR.CurMF, NewReg); 1056 MachineInstr *MI; 1057 MachineBasicBlock &BB = *I.getParent(); 1058 if (Val == 0) { 1059 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1060 .addDef(NewReg) 1061 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 1062 } else { 1063 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1064 .addDef(NewReg) 1065 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 1066 .addImm(APInt(32, Val).getZExtValue()); 1067 } 1068 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 1069 } 1070 return NewReg; 1071 } 1072 1073 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 1074 const SPIRVType *ResType, 1075 MachineInstr &I) const { 1076 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 1077 return selectCmp(ResVReg, ResType, CmpOp, I); 1078 } 1079 1080 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 1081 MachineInstr &I) const { 1082 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1083 return GR.getOrCreateConsIntVector(0, I, ResType, TII); 1084 return GR.getOrCreateConstInt(0, I, ResType, TII); 1085 } 1086 1087 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 1088 const SPIRVType *ResType, 1089 MachineInstr &I) const { 1090 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1091 APInt One = 1092 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 1093 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1094 return GR.getOrCreateConsIntVector(One.getZExtValue(), I, ResType, TII); 1095 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 1096 } 1097 1098 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 1099 const SPIRVType *ResType, 1100 MachineInstr &I, 1101 bool IsSigned) const { 1102 // To extend a bool, we need to use OpSelect between constants. 1103 Register ZeroReg = buildZerosVal(ResType, I); 1104 Register OneReg = buildOnesVal(IsSigned, ResType, I); 1105 bool IsScalarBool = 1106 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 1107 unsigned Opcode = 1108 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 1109 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1110 .addDef(ResVReg) 1111 .addUse(GR.getSPIRVTypeID(ResType)) 1112 .addUse(I.getOperand(1).getReg()) 1113 .addUse(OneReg) 1114 .addUse(ZeroReg) 1115 .constrainAllUses(TII, TRI, RBI); 1116 } 1117 1118 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 1119 const SPIRVType *ResType, 1120 MachineInstr &I, bool IsSigned, 1121 unsigned Opcode) const { 1122 Register SrcReg = I.getOperand(1).getReg(); 1123 // We can convert bool value directly to float type without OpConvert*ToF, 1124 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 1125 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 1126 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1127 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 1128 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 1129 const unsigned NumElts = ResType->getOperand(2).getImm(); 1130 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 1131 } 1132 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1133 selectSelect(SrcReg, TmpType, I, false); 1134 } 1135 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 1136 } 1137 1138 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 1139 const SPIRVType *ResType, 1140 MachineInstr &I, bool IsSigned) const { 1141 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) 1142 return selectSelect(ResVReg, ResType, I, IsSigned); 1143 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1144 return selectUnOp(ResVReg, ResType, I, Opcode); 1145 } 1146 1147 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 1148 Register ResVReg, 1149 MachineInstr &I, 1150 const SPIRVType *IntTy, 1151 const SPIRVType *BoolTy) const { 1152 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 1153 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1154 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 1155 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 1156 Register Zero = buildZerosVal(IntTy, I); 1157 Register One = buildOnesVal(false, IntTy, I); 1158 MachineBasicBlock &BB = *I.getParent(); 1159 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1160 .addDef(BitIntReg) 1161 .addUse(GR.getSPIRVTypeID(IntTy)) 1162 .addUse(IntReg) 1163 .addUse(One) 1164 .constrainAllUses(TII, TRI, RBI); 1165 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1166 .addDef(ResVReg) 1167 .addUse(GR.getSPIRVTypeID(BoolTy)) 1168 .addUse(BitIntReg) 1169 .addUse(Zero) 1170 .constrainAllUses(TII, TRI, RBI); 1171 } 1172 1173 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 1174 const SPIRVType *ResType, 1175 MachineInstr &I) const { 1176 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { 1177 Register IntReg = I.getOperand(1).getReg(); 1178 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 1179 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 1180 } 1181 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 1182 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1183 return selectUnOp(ResVReg, ResType, I, Opcode); 1184 } 1185 1186 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 1187 const SPIRVType *ResType, 1188 const APInt &Imm, 1189 MachineInstr &I) const { 1190 unsigned TyOpcode = ResType->getOpcode(); 1191 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 1192 MachineBasicBlock &BB = *I.getParent(); 1193 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 1194 Imm.isZero()) 1195 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1196 .addDef(ResVReg) 1197 .addUse(GR.getSPIRVTypeID(ResType)) 1198 .constrainAllUses(TII, TRI, RBI); 1199 if (TyOpcode == SPIRV::OpTypeInt) { 1200 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 1201 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 1202 if (Reg == ResVReg) 1203 return true; 1204 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1205 .addDef(ResVReg) 1206 .addUse(Reg) 1207 .constrainAllUses(TII, TRI, RBI); 1208 } 1209 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1210 .addDef(ResVReg) 1211 .addUse(GR.getSPIRVTypeID(ResType)); 1212 // <=32-bit integers should be caught by the sdag pattern. 1213 assert(Imm.getBitWidth() > 32); 1214 addNumImm(Imm, MIB); 1215 return MIB.constrainAllUses(TII, TRI, RBI); 1216 } 1217 1218 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 1219 const SPIRVType *ResType, 1220 MachineInstr &I) const { 1221 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1222 .addDef(ResVReg) 1223 .addUse(GR.getSPIRVTypeID(ResType)) 1224 .constrainAllUses(TII, TRI, RBI); 1225 } 1226 1227 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1228 assert(MO.isReg()); 1229 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1230 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) 1231 return false; 1232 assert(TypeInst->getOperand(1).isReg()); 1233 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1234 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 1235 } 1236 1237 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1238 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1239 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1240 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 1241 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 1242 } 1243 1244 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 1245 const SPIRVType *ResType, 1246 MachineInstr &I) const { 1247 MachineBasicBlock &BB = *I.getParent(); 1248 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 1249 .addDef(ResVReg) 1250 .addUse(GR.getSPIRVTypeID(ResType)) 1251 // object to insert 1252 .addUse(I.getOperand(3).getReg()) 1253 // composite to insert into 1254 .addUse(I.getOperand(2).getReg()); 1255 for (unsigned i = 4; i < I.getNumOperands(); i++) 1256 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1257 return MIB.constrainAllUses(TII, TRI, RBI); 1258 } 1259 1260 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 1261 const SPIRVType *ResType, 1262 MachineInstr &I) const { 1263 MachineBasicBlock &BB = *I.getParent(); 1264 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1265 .addDef(ResVReg) 1266 .addUse(GR.getSPIRVTypeID(ResType)) 1267 .addUse(I.getOperand(2).getReg()); 1268 for (unsigned i = 3; i < I.getNumOperands(); i++) 1269 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1270 return MIB.constrainAllUses(TII, TRI, RBI); 1271 } 1272 1273 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 1274 const SPIRVType *ResType, 1275 MachineInstr &I) const { 1276 if (isImm(I.getOperand(4), MRI)) 1277 return selectInsertVal(ResVReg, ResType, I); 1278 MachineBasicBlock &BB = *I.getParent(); 1279 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 1280 .addDef(ResVReg) 1281 .addUse(GR.getSPIRVTypeID(ResType)) 1282 .addUse(I.getOperand(2).getReg()) 1283 .addUse(I.getOperand(3).getReg()) 1284 .addUse(I.getOperand(4).getReg()) 1285 .constrainAllUses(TII, TRI, RBI); 1286 } 1287 1288 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 1289 const SPIRVType *ResType, 1290 MachineInstr &I) const { 1291 if (isImm(I.getOperand(3), MRI)) 1292 return selectExtractVal(ResVReg, ResType, I); 1293 MachineBasicBlock &BB = *I.getParent(); 1294 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 1295 .addDef(ResVReg) 1296 .addUse(GR.getSPIRVTypeID(ResType)) 1297 .addUse(I.getOperand(2).getReg()) 1298 .addUse(I.getOperand(3).getReg()) 1299 .constrainAllUses(TII, TRI, RBI); 1300 } 1301 1302 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 1303 const SPIRVType *ResType, 1304 MachineInstr &I) const { 1305 const bool IsGEPInBounds = I.getOperand(2).getImm(); 1306 1307 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 1308 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 1309 // we have to use Op[InBounds]AccessChain. 1310 const unsigned Opcode = STI.isVulkanEnv() 1311 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 1312 : SPIRV::OpAccessChain) 1313 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 1314 : SPIRV::OpPtrAccessChain); 1315 1316 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1317 .addDef(ResVReg) 1318 .addUse(GR.getSPIRVTypeID(ResType)) 1319 // Object to get a pointer to. 1320 .addUse(I.getOperand(3).getReg()); 1321 // Adding indices. 1322 const unsigned StartingIndex = 1323 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 1324 ? 5 1325 : 4; 1326 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 1327 Res.addUse(I.getOperand(i).getReg()); 1328 return Res.constrainAllUses(TII, TRI, RBI); 1329 } 1330 1331 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 1332 const SPIRVType *ResType, 1333 MachineInstr &I) const { 1334 MachineBasicBlock &BB = *I.getParent(); 1335 switch (cast<GIntrinsic>(I).getIntrinsicID()) { 1336 case Intrinsic::spv_load: 1337 return selectLoad(ResVReg, ResType, I); 1338 case Intrinsic::spv_store: 1339 return selectStore(I); 1340 case Intrinsic::spv_extractv: 1341 return selectExtractVal(ResVReg, ResType, I); 1342 case Intrinsic::spv_insertv: 1343 return selectInsertVal(ResVReg, ResType, I); 1344 case Intrinsic::spv_extractelt: 1345 return selectExtractElt(ResVReg, ResType, I); 1346 case Intrinsic::spv_insertelt: 1347 return selectInsertElt(ResVReg, ResType, I); 1348 case Intrinsic::spv_gep: 1349 return selectGEP(ResVReg, ResType, I); 1350 case Intrinsic::spv_unref_global: 1351 case Intrinsic::spv_init_global: { 1352 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 1353 MachineInstr *Init = I.getNumExplicitOperands() > 2 1354 ? MRI->getVRegDef(I.getOperand(2).getReg()) 1355 : nullptr; 1356 assert(MI); 1357 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 1358 } 1359 case Intrinsic::spv_undef: { 1360 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1361 .addDef(ResVReg) 1362 .addUse(GR.getSPIRVTypeID(ResType)); 1363 return MIB.constrainAllUses(TII, TRI, RBI); 1364 } 1365 case Intrinsic::spv_const_composite: { 1366 // If no values are attached, the composite is null constant. 1367 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 1368 unsigned Opcode = 1369 IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; 1370 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1371 .addDef(ResVReg) 1372 .addUse(GR.getSPIRVTypeID(ResType)); 1373 // skip type MD node we already used when generated assign.type for this 1374 if (!IsNull) { 1375 for (unsigned i = I.getNumExplicitDefs() + 1; 1376 i < I.getNumExplicitOperands(); ++i) { 1377 MIB.addUse(I.getOperand(i).getReg()); 1378 } 1379 } 1380 return MIB.constrainAllUses(TII, TRI, RBI); 1381 } 1382 case Intrinsic::spv_assign_name: { 1383 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 1384 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 1385 for (unsigned i = I.getNumExplicitDefs() + 2; 1386 i < I.getNumExplicitOperands(); ++i) { 1387 MIB.addImm(I.getOperand(i).getImm()); 1388 } 1389 return MIB.constrainAllUses(TII, TRI, RBI); 1390 } 1391 case Intrinsic::spv_switch: { 1392 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 1393 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 1394 if (I.getOperand(i).isReg()) 1395 MIB.addReg(I.getOperand(i).getReg()); 1396 else if (I.getOperand(i).isCImm()) 1397 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 1398 else if (I.getOperand(i).isMBB()) 1399 MIB.addMBB(I.getOperand(i).getMBB()); 1400 else 1401 llvm_unreachable("Unexpected OpSwitch operand"); 1402 } 1403 return MIB.constrainAllUses(TII, TRI, RBI); 1404 } 1405 case Intrinsic::spv_cmpxchg: 1406 return selectAtomicCmpXchg(ResVReg, ResType, I); 1407 case Intrinsic::spv_unreachable: 1408 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 1409 break; 1410 case Intrinsic::spv_alloca: 1411 return selectFrameIndex(ResVReg, ResType, I); 1412 case Intrinsic::spv_assume: 1413 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 1414 .addUse(I.getOperand(1).getReg()); 1415 break; 1416 case Intrinsic::spv_expect: 1417 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 1418 .addDef(ResVReg) 1419 .addUse(GR.getSPIRVTypeID(ResType)) 1420 .addUse(I.getOperand(2).getReg()) 1421 .addUse(I.getOperand(3).getReg()); 1422 break; 1423 default: 1424 llvm_unreachable("Intrinsic selection not implemented"); 1425 } 1426 return true; 1427 } 1428 1429 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 1430 const SPIRVType *ResType, 1431 MachineInstr &I) const { 1432 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1433 .addDef(ResVReg) 1434 .addUse(GR.getSPIRVTypeID(ResType)) 1435 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 1436 .constrainAllUses(TII, TRI, RBI); 1437 } 1438 1439 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 1440 // InstructionSelector walks backwards through the instructions. We can use 1441 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 1442 // first, so can generate an OpBranchConditional here. If there is no 1443 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 1444 const MachineInstr *PrevI = I.getPrevNode(); 1445 MachineBasicBlock &MBB = *I.getParent(); 1446 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 1447 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1448 .addUse(PrevI->getOperand(0).getReg()) 1449 .addMBB(PrevI->getOperand(1).getMBB()) 1450 .addMBB(I.getOperand(0).getMBB()) 1451 .constrainAllUses(TII, TRI, RBI); 1452 } 1453 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 1454 .addMBB(I.getOperand(0).getMBB()) 1455 .constrainAllUses(TII, TRI, RBI); 1456 } 1457 1458 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 1459 // InstructionSelector walks backwards through the instructions. For an 1460 // explicit conditional branch with no fallthrough, we use both a G_BR and a 1461 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 1462 // generate the OpBranchConditional in selectBranch above. 1463 // 1464 // If an OpBranchConditional has been generated, we simply return, as the work 1465 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 1466 // implicit fallthrough to the next basic block, so we need to create an 1467 // OpBranchConditional with an explicit "false" argument pointing to the next 1468 // basic block that LLVM would fall through to. 1469 const MachineInstr *NextI = I.getNextNode(); 1470 // Check if this has already been successfully selected. 1471 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 1472 return true; 1473 // Must be relying on implicit block fallthrough, so generate an 1474 // OpBranchConditional with the "next" basic block as the "false" target. 1475 MachineBasicBlock &MBB = *I.getParent(); 1476 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 1477 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 1478 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1479 .addUse(I.getOperand(0).getReg()) 1480 .addMBB(I.getOperand(1).getMBB()) 1481 .addMBB(NextMBB) 1482 .constrainAllUses(TII, TRI, RBI); 1483 } 1484 1485 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 1486 const SPIRVType *ResType, 1487 MachineInstr &I) const { 1488 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 1489 .addDef(ResVReg) 1490 .addUse(GR.getSPIRVTypeID(ResType)); 1491 const unsigned NumOps = I.getNumOperands(); 1492 for (unsigned i = 1; i < NumOps; i += 2) { 1493 MIB.addUse(I.getOperand(i + 0).getReg()); 1494 MIB.addMBB(I.getOperand(i + 1).getMBB()); 1495 } 1496 return MIB.constrainAllUses(TII, TRI, RBI); 1497 } 1498 1499 bool SPIRVInstructionSelector::selectGlobalValue( 1500 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 1501 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 1502 MachineIRBuilder MIRBuilder(I); 1503 const GlobalValue *GV = I.getOperand(1).getGlobal(); 1504 Type *GVType = GV->getValueType(); 1505 SPIRVType *PointerBaseType; 1506 if (GVType->isArrayTy()) { 1507 SPIRVType *ArrayElementType = 1508 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 1509 SPIRV::AccessQualifier::ReadWrite, false); 1510 PointerBaseType = GR.getOrCreateSPIRVArrayType( 1511 ArrayElementType, GVType->getArrayNumElements(), I, TII); 1512 } else { 1513 PointerBaseType = GR.getOrCreateSPIRVType( 1514 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1515 } 1516 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 1517 PointerBaseType, I, TII, 1518 addressSpaceToStorageClass(GV->getAddressSpace())); 1519 std::string GlobalIdent = GV->getGlobalIdentifier(); 1520 // We have functions as operands in tests with blocks of instruction e.g. in 1521 // transcoding/global_block.ll. These operands are not used and should be 1522 // substituted by zero constants. Their type is expected to be always 1523 // OpTypePointer Function %uchar. 1524 if (isa<Function>(GV)) { 1525 const Constant *ConstVal = GV; 1526 MachineBasicBlock &BB = *I.getParent(); 1527 Register NewReg = GR.find(ConstVal, GR.CurMF); 1528 if (!NewReg.isValid()) { 1529 Register NewReg = ResVReg; 1530 GR.add(ConstVal, GR.CurMF, NewReg); 1531 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1532 .addDef(NewReg) 1533 .addUse(GR.getSPIRVTypeID(ResType)) 1534 .constrainAllUses(TII, TRI, RBI); 1535 } 1536 assert(NewReg != ResVReg); 1537 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1538 .addDef(ResVReg) 1539 .addUse(NewReg) 1540 .constrainAllUses(TII, TRI, RBI); 1541 } 1542 auto GlobalVar = cast<GlobalVariable>(GV); 1543 assert(GlobalVar->getName() != "llvm.global.annotations"); 1544 1545 bool HasInit = GlobalVar->hasInitializer() && 1546 !isa<UndefValue>(GlobalVar->getInitializer()); 1547 // Skip empty declaration for GVs with initilaizers till we get the decl with 1548 // passed initializer. 1549 if (HasInit && !Init) 1550 return true; 1551 1552 unsigned AddrSpace = GV->getAddressSpace(); 1553 SPIRV::StorageClass::StorageClass Storage = 1554 addressSpaceToStorageClass(AddrSpace); 1555 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 1556 Storage != SPIRV::StorageClass::Function; 1557 SPIRV::LinkageType::LinkageType LnkType = 1558 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 1559 ? SPIRV::LinkageType::Import 1560 : SPIRV::LinkageType::Export; 1561 1562 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 1563 Storage, Init, GlobalVar->isConstant(), 1564 HasLnkTy, LnkType, MIRBuilder, true); 1565 return Reg.isValid(); 1566 } 1567 1568 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 1569 const SPIRVType *ResType, 1570 MachineInstr &I) const { 1571 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 1572 return selectExtInst(ResVReg, ResType, I, CL::log10); 1573 } 1574 1575 // There is no log10 instruction in the GLSL Extended Instruction set, so it 1576 // is implemented as: 1577 // log10(x) = log2(x) * (1 / log2(10)) 1578 // = log2(x) * 0.30103 1579 1580 MachineIRBuilder MIRBuilder(I); 1581 MachineBasicBlock &BB = *I.getParent(); 1582 1583 // Build log2(x). 1584 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1585 bool Result = 1586 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1587 .addDef(VarReg) 1588 .addUse(GR.getSPIRVTypeID(ResType)) 1589 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1590 .addImm(GL::Log2) 1591 .add(I.getOperand(1)) 1592 .constrainAllUses(TII, TRI, RBI); 1593 1594 // Build 0.30103. 1595 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 1596 ResType->getOpcode() == SPIRV::OpTypeFloat); 1597 // TODO: Add matrix implementation once supported by the HLSL frontend. 1598 const SPIRVType *SpirvScalarType = 1599 ResType->getOpcode() == SPIRV::OpTypeVector 1600 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 1601 : ResType; 1602 Register ScaleReg = 1603 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 1604 1605 // Multiply log2(x) by 0.30103 to get log10(x) result. 1606 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 1607 ? SPIRV::OpVectorTimesScalar 1608 : SPIRV::OpFMulS; 1609 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1610 .addDef(ResVReg) 1611 .addUse(GR.getSPIRVTypeID(ResType)) 1612 .addUse(VarReg) 1613 .addUse(ScaleReg) 1614 .constrainAllUses(TII, TRI, RBI); 1615 1616 return Result; 1617 } 1618 1619 namespace llvm { 1620 InstructionSelector * 1621 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 1622 const SPIRVSubtarget &Subtarget, 1623 const RegisterBankInfo &RBI) { 1624 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 1625 } 1626 } // namespace llvm 1627