1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineRegisterInfo.h" 30 #include "llvm/IR/IntrinsicsSPIRV.h" 31 #include "llvm/Support/Debug.h" 32 33 #define DEBUG_TYPE "spirv-isel" 34 35 using namespace llvm; 36 namespace CL = SPIRV::OpenCLExtInst; 37 namespace GL = SPIRV::GLSLExtInst; 38 39 using ExtInstList = 40 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 41 42 namespace { 43 44 #define GET_GLOBALISEL_PREDICATE_BITSET 45 #include "SPIRVGenGlobalISel.inc" 46 #undef GET_GLOBALISEL_PREDICATE_BITSET 47 48 class SPIRVInstructionSelector : public InstructionSelector { 49 const SPIRVSubtarget &STI; 50 const SPIRVInstrInfo &TII; 51 const SPIRVRegisterInfo &TRI; 52 const RegisterBankInfo &RBI; 53 SPIRVGlobalRegistry &GR; 54 MachineRegisterInfo *MRI; 55 56 /// We need to keep track of the number we give to anonymous global values to 57 /// generate the same name every time when this is needed. 58 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 59 60 public: 61 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 62 const SPIRVSubtarget &ST, 63 const RegisterBankInfo &RBI); 64 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 65 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 66 BlockFrequencyInfo *BFI) override; 67 // Common selection code. Instruction-specific selection occurs in spvSelect. 68 bool select(MachineInstr &I) override; 69 static const char *getName() { return DEBUG_TYPE; } 70 71 #define GET_GLOBALISEL_PREDICATES_DECL 72 #include "SPIRVGenGlobalISel.inc" 73 #undef GET_GLOBALISEL_PREDICATES_DECL 74 75 #define GET_GLOBALISEL_TEMPORARIES_DECL 76 #include "SPIRVGenGlobalISel.inc" 77 #undef GET_GLOBALISEL_TEMPORARIES_DECL 78 79 private: 80 // tblgen-erated 'select' implementation, used as the initial selector for 81 // the patterns that don't require complex C++. 82 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 83 84 // All instruction-specific selection that didn't happen in "select()". 85 // Is basically a large Switch/Case delegating to all other select method. 86 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 87 MachineInstr &I) const; 88 89 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 90 const MachineInstr *Init = nullptr) const; 91 92 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 93 MachineInstr &I, Register SrcReg, 94 unsigned Opcode) const; 95 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 96 unsigned Opcode) const; 97 98 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 99 MachineInstr &I) const; 100 bool selectStore(MachineInstr &I) const; 101 102 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 103 104 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 105 MachineInstr &I, unsigned NewOpcode, 106 unsigned NegateOpcode = 0) const; 107 108 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 109 MachineInstr &I) const; 110 111 bool selectFence(MachineInstr &I) const; 112 113 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 114 MachineInstr &I) const; 115 116 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 117 MachineInstr &I) const; 118 119 bool selectConstVector(Register ResVReg, const SPIRVType *ResType, 120 MachineInstr &I) const; 121 122 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 123 unsigned comparisonOpcode, MachineInstr &I) const; 124 125 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 126 MachineInstr &I) const; 127 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 128 MachineInstr &I) const; 129 130 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 131 int OpIdx) const; 132 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 133 int OpIdx) const; 134 135 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 136 MachineInstr &I) const; 137 138 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 139 bool IsSigned) const; 140 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 141 bool IsSigned, unsigned Opcode) const; 142 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 143 bool IsSigned) const; 144 145 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 146 MachineInstr &I) const; 147 148 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 149 const SPIRVType *intTy, const SPIRVType *boolTy) const; 150 151 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 152 MachineInstr &I) const; 153 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 154 MachineInstr &I) const; 155 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 156 MachineInstr &I) const; 157 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 158 MachineInstr &I) const; 159 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 160 MachineInstr &I) const; 161 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 162 MachineInstr &I) const; 163 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 164 MachineInstr &I) const; 165 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 166 MachineInstr &I) const; 167 168 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 169 MachineInstr &I) const; 170 171 bool selectBranch(MachineInstr &I) const; 172 bool selectBranchCond(MachineInstr &I) const; 173 174 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 175 MachineInstr &I) const; 176 177 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 178 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 179 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 180 MachineInstr &I, CL::OpenCLExtInst CLInst, 181 GL::GLSLExtInst GLInst) const; 182 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 183 MachineInstr &I, const ExtInstList &ExtInsts) const; 184 185 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 186 MachineInstr &I) const; 187 188 Register buildI32Constant(uint32_t Val, MachineInstr &I, 189 const SPIRVType *ResType = nullptr) const; 190 191 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 192 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 }; 195 196 } // end anonymous namespace 197 198 #define GET_GLOBALISEL_IMPL 199 #include "SPIRVGenGlobalISel.inc" 200 #undef GET_GLOBALISEL_IMPL 201 202 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 203 const SPIRVSubtarget &ST, 204 const RegisterBankInfo &RBI) 205 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 206 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 207 #define GET_GLOBALISEL_PREDICATES_INIT 208 #include "SPIRVGenGlobalISel.inc" 209 #undef GET_GLOBALISEL_PREDICATES_INIT 210 #define GET_GLOBALISEL_TEMPORARIES_INIT 211 #include "SPIRVGenGlobalISel.inc" 212 #undef GET_GLOBALISEL_TEMPORARIES_INIT 213 { 214 } 215 216 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 217 CodeGenCoverage *CoverageInfo, 218 ProfileSummaryInfo *PSI, 219 BlockFrequencyInfo *BFI) { 220 MRI = &MF.getRegInfo(); 221 GR.setCurrentFunc(MF); 222 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 223 } 224 225 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 226 227 // Defined in SPIRVLegalizerInfo.cpp. 228 extern bool isTypeFoldingSupported(unsigned Opcode); 229 230 bool SPIRVInstructionSelector::select(MachineInstr &I) { 231 assert(I.getParent() && "Instruction should be in a basic block!"); 232 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 233 234 Register Opcode = I.getOpcode(); 235 // If it's not a GMIR instruction, we've selected it already. 236 if (!isPreISelGenericOpcode(Opcode)) { 237 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 238 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); 239 if (isTypeFoldingSupported(Def->getOpcode())) { 240 auto Res = selectImpl(I, *CoverageInfo); 241 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 242 if (Res) 243 return Res; 244 } 245 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); 246 I.removeFromParent(); 247 return true; 248 } else if (I.getNumDefs() == 1) { 249 // Make all vregs 32 bits (for SPIR-V IDs). 250 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); 251 } 252 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 253 } 254 255 if (I.getNumOperands() != I.getNumExplicitOperands()) { 256 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 257 return false; 258 } 259 260 // Common code for getting return reg+type, and removing selected instr 261 // from parent occurs here. Instr-specific selection happens in spvSelect(). 262 bool HasDefs = I.getNumDefs() > 0; 263 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 264 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 265 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 266 if (spvSelect(ResVReg, ResType, I)) { 267 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). 268 MRI->setType(ResVReg, LLT::scalar(32)); 269 I.removeFromParent(); 270 return true; 271 } 272 return false; 273 } 274 275 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 276 const SPIRVType *ResType, 277 MachineInstr &I) const { 278 assert(!isTypeFoldingSupported(I.getOpcode()) || 279 I.getOpcode() == TargetOpcode::G_CONSTANT); 280 const unsigned Opcode = I.getOpcode(); 281 switch (Opcode) { 282 case TargetOpcode::G_CONSTANT: 283 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 284 I); 285 case TargetOpcode::G_GLOBAL_VALUE: 286 return selectGlobalValue(ResVReg, I); 287 case TargetOpcode::G_IMPLICIT_DEF: 288 return selectOpUndef(ResVReg, ResType, I); 289 case TargetOpcode::G_FREEZE: 290 return selectFreeze(ResVReg, ResType, I); 291 292 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 293 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 294 return selectIntrinsic(ResVReg, ResType, I); 295 case TargetOpcode::G_BITREVERSE: 296 return selectBitreverse(ResVReg, ResType, I); 297 298 case TargetOpcode::G_BUILD_VECTOR: 299 return selectConstVector(ResVReg, ResType, I); 300 301 case TargetOpcode::G_SHUFFLE_VECTOR: { 302 MachineBasicBlock &BB = *I.getParent(); 303 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 304 .addDef(ResVReg) 305 .addUse(GR.getSPIRVTypeID(ResType)) 306 .addUse(I.getOperand(1).getReg()) 307 .addUse(I.getOperand(2).getReg()); 308 for (auto V : I.getOperand(3).getShuffleMask()) 309 MIB.addImm(V); 310 return MIB.constrainAllUses(TII, TRI, RBI); 311 } 312 case TargetOpcode::G_MEMMOVE: 313 case TargetOpcode::G_MEMCPY: 314 case TargetOpcode::G_MEMSET: 315 return selectMemOperation(ResVReg, I); 316 317 case TargetOpcode::G_ICMP: 318 return selectICmp(ResVReg, ResType, I); 319 case TargetOpcode::G_FCMP: 320 return selectFCmp(ResVReg, ResType, I); 321 322 case TargetOpcode::G_FRAME_INDEX: 323 return selectFrameIndex(ResVReg, ResType, I); 324 325 case TargetOpcode::G_LOAD: 326 return selectLoad(ResVReg, ResType, I); 327 case TargetOpcode::G_STORE: 328 return selectStore(I); 329 330 case TargetOpcode::G_BR: 331 return selectBranch(I); 332 case TargetOpcode::G_BRCOND: 333 return selectBranchCond(I); 334 335 case TargetOpcode::G_PHI: 336 return selectPhi(ResVReg, ResType, I); 337 338 case TargetOpcode::G_FPTOSI: 339 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 340 case TargetOpcode::G_FPTOUI: 341 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 342 343 case TargetOpcode::G_SITOFP: 344 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 345 case TargetOpcode::G_UITOFP: 346 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 347 348 case TargetOpcode::G_CTPOP: 349 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 350 case TargetOpcode::G_SMIN: 351 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 352 case TargetOpcode::G_UMIN: 353 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 354 355 case TargetOpcode::G_SMAX: 356 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 357 case TargetOpcode::G_UMAX: 358 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 359 360 case TargetOpcode::G_FMA: 361 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 362 363 case TargetOpcode::G_FPOW: 364 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 365 case TargetOpcode::G_FPOWI: 366 return selectExtInst(ResVReg, ResType, I, CL::pown); 367 368 case TargetOpcode::G_FEXP: 369 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 370 case TargetOpcode::G_FEXP2: 371 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 372 373 case TargetOpcode::G_FLOG: 374 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 375 case TargetOpcode::G_FLOG2: 376 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 377 case TargetOpcode::G_FLOG10: 378 return selectLog10(ResVReg, ResType, I); 379 380 case TargetOpcode::G_FABS: 381 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 382 case TargetOpcode::G_ABS: 383 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 384 385 case TargetOpcode::G_FMINNUM: 386 case TargetOpcode::G_FMINIMUM: 387 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin); 388 case TargetOpcode::G_FMAXNUM: 389 case TargetOpcode::G_FMAXIMUM: 390 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax); 391 392 case TargetOpcode::G_FCOPYSIGN: 393 return selectExtInst(ResVReg, ResType, I, CL::copysign); 394 395 case TargetOpcode::G_FCEIL: 396 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 397 case TargetOpcode::G_FFLOOR: 398 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 399 400 case TargetOpcode::G_FCOS: 401 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 402 case TargetOpcode::G_FSIN: 403 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 404 405 case TargetOpcode::G_FSQRT: 406 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 407 408 case TargetOpcode::G_CTTZ: 409 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 410 return selectExtInst(ResVReg, ResType, I, CL::ctz); 411 case TargetOpcode::G_CTLZ: 412 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 413 return selectExtInst(ResVReg, ResType, I, CL::clz); 414 415 case TargetOpcode::G_INTRINSIC_ROUND: 416 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 417 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 418 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 419 case TargetOpcode::G_INTRINSIC_TRUNC: 420 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 421 case TargetOpcode::G_FRINT: 422 case TargetOpcode::G_FNEARBYINT: 423 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 424 425 case TargetOpcode::G_SMULH: 426 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 427 case TargetOpcode::G_UMULH: 428 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 429 430 case TargetOpcode::G_SEXT: 431 return selectExt(ResVReg, ResType, I, true); 432 case TargetOpcode::G_ANYEXT: 433 case TargetOpcode::G_ZEXT: 434 return selectExt(ResVReg, ResType, I, false); 435 case TargetOpcode::G_TRUNC: 436 return selectTrunc(ResVReg, ResType, I); 437 case TargetOpcode::G_FPTRUNC: 438 case TargetOpcode::G_FPEXT: 439 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 440 441 case TargetOpcode::G_PTRTOINT: 442 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 443 case TargetOpcode::G_INTTOPTR: 444 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 445 case TargetOpcode::G_BITCAST: 446 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 447 case TargetOpcode::G_ADDRSPACE_CAST: 448 return selectAddrSpaceCast(ResVReg, ResType, I); 449 case TargetOpcode::G_PTR_ADD: { 450 // Currently, we get G_PTR_ADD only as a result of translating 451 // global variables, initialized with constant expressions like GV + Const 452 // (see test opencl/basic/progvar_prog_scope_init.ll). 453 // TODO: extend the handler once we have other cases. 454 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 455 Register GV = I.getOperand(1).getReg(); 456 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 457 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 458 (*II).getOpcode() == TargetOpcode::COPY || 459 (*II).getOpcode() == SPIRV::OpVariable) && 460 isImm(I.getOperand(2), MRI)); 461 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 462 MachineBasicBlock &BB = *I.getParent(); 463 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 464 .addDef(ResVReg) 465 .addUse(GR.getSPIRVTypeID(ResType)) 466 .addImm(static_cast<uint32_t>( 467 SPIRV::Opcode::InBoundsPtrAccessChain)) 468 .addUse(GV) 469 .addUse(Idx) 470 .addUse(I.getOperand(2).getReg()); 471 return MIB.constrainAllUses(TII, TRI, RBI); 472 } 473 474 case TargetOpcode::G_ATOMICRMW_OR: 475 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 476 case TargetOpcode::G_ATOMICRMW_ADD: 477 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 478 case TargetOpcode::G_ATOMICRMW_AND: 479 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 480 case TargetOpcode::G_ATOMICRMW_MAX: 481 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 482 case TargetOpcode::G_ATOMICRMW_MIN: 483 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 484 case TargetOpcode::G_ATOMICRMW_SUB: 485 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 486 case TargetOpcode::G_ATOMICRMW_XOR: 487 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 488 case TargetOpcode::G_ATOMICRMW_UMAX: 489 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 490 case TargetOpcode::G_ATOMICRMW_UMIN: 491 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 492 case TargetOpcode::G_ATOMICRMW_XCHG: 493 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 494 case TargetOpcode::G_ATOMIC_CMPXCHG: 495 return selectAtomicCmpXchg(ResVReg, ResType, I); 496 497 case TargetOpcode::G_ATOMICRMW_FADD: 498 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 499 case TargetOpcode::G_ATOMICRMW_FSUB: 500 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 501 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 502 SPIRV::OpFNegate); 503 case TargetOpcode::G_ATOMICRMW_FMIN: 504 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 505 case TargetOpcode::G_ATOMICRMW_FMAX: 506 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 507 508 case TargetOpcode::G_FENCE: 509 return selectFence(I); 510 511 default: 512 return false; 513 } 514 } 515 516 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 517 const SPIRVType *ResType, 518 MachineInstr &I, 519 CL::OpenCLExtInst CLInst) const { 520 return selectExtInst(ResVReg, ResType, I, 521 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 522 } 523 524 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 525 const SPIRVType *ResType, 526 MachineInstr &I, 527 CL::OpenCLExtInst CLInst, 528 GL::GLSLExtInst GLInst) const { 529 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 530 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 531 return selectExtInst(ResVReg, ResType, I, ExtInsts); 532 } 533 534 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 535 const SPIRVType *ResType, 536 MachineInstr &I, 537 const ExtInstList &Insts) const { 538 539 for (const auto &Ex : Insts) { 540 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 541 uint32_t Opcode = Ex.second; 542 if (STI.canUseExtInstSet(Set)) { 543 MachineBasicBlock &BB = *I.getParent(); 544 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 545 .addDef(ResVReg) 546 .addUse(GR.getSPIRVTypeID(ResType)) 547 .addImm(static_cast<uint32_t>(Set)) 548 .addImm(Opcode); 549 const unsigned NumOps = I.getNumOperands(); 550 for (unsigned i = 1; i < NumOps; ++i) 551 MIB.add(I.getOperand(i)); 552 return MIB.constrainAllUses(TII, TRI, RBI); 553 } 554 } 555 return false; 556 } 557 558 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 559 const SPIRVType *ResType, 560 MachineInstr &I, 561 Register SrcReg, 562 unsigned Opcode) const { 563 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 564 .addDef(ResVReg) 565 .addUse(GR.getSPIRVTypeID(ResType)) 566 .addUse(SrcReg) 567 .constrainAllUses(TII, TRI, RBI); 568 } 569 570 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 571 const SPIRVType *ResType, 572 MachineInstr &I, 573 unsigned Opcode) const { 574 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 575 Opcode); 576 } 577 578 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) { 579 switch (Ord) { 580 case SyncScope::SingleThread: 581 return SPIRV::Scope::Invocation; 582 case SyncScope::System: 583 return SPIRV::Scope::Device; 584 default: 585 llvm_unreachable("Unsupported synchronization Scope ID."); 586 } 587 } 588 589 static void addMemoryOperands(MachineMemOperand *MemOp, 590 MachineInstrBuilder &MIB) { 591 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 592 if (MemOp->isVolatile()) 593 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 594 if (MemOp->isNonTemporal()) 595 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 596 if (MemOp->getAlign().value()) 597 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 598 599 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 600 MIB.addImm(SpvMemOp); 601 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 602 MIB.addImm(MemOp->getAlign().value()); 603 } 604 } 605 606 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 607 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 608 if (Flags & MachineMemOperand::Flags::MOVolatile) 609 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 610 if (Flags & MachineMemOperand::Flags::MONonTemporal) 611 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 612 613 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 614 MIB.addImm(SpvMemOp); 615 } 616 617 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 618 const SPIRVType *ResType, 619 MachineInstr &I) const { 620 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 621 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 622 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 623 .addDef(ResVReg) 624 .addUse(GR.getSPIRVTypeID(ResType)) 625 .addUse(Ptr); 626 if (!I.getNumMemOperands()) { 627 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 628 I.getOpcode() == 629 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 630 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 631 } else { 632 addMemoryOperands(*I.memoperands_begin(), MIB); 633 } 634 return MIB.constrainAllUses(TII, TRI, RBI); 635 } 636 637 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 638 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 639 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 640 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 641 MachineBasicBlock &BB = *I.getParent(); 642 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 643 .addUse(Ptr) 644 .addUse(StoreVal); 645 if (!I.getNumMemOperands()) { 646 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 647 I.getOpcode() == 648 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 649 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 650 } else { 651 addMemoryOperands(*I.memoperands_begin(), MIB); 652 } 653 return MIB.constrainAllUses(TII, TRI, RBI); 654 } 655 656 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 657 MachineInstr &I) const { 658 MachineBasicBlock &BB = *I.getParent(); 659 Register SrcReg = I.getOperand(1).getReg(); 660 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 661 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 662 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 663 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 664 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 665 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 666 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); 667 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 668 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 669 // TODO: check if we have such GV, add init, use buildGlobalVariable. 670 Type *LLVMArrTy = ArrayType::get( 671 IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); 672 GlobalVariable *GV = 673 new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); 674 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 675 GR.add(GV, GR.CurMF, VarReg); 676 677 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 678 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 679 .addDef(VarReg) 680 .addUse(GR.getSPIRVTypeID(VarTy)) 681 .addImm(SPIRV::StorageClass::UniformConstant) 682 .addUse(Const) 683 .constrainAllUses(TII, TRI, RBI); 684 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 685 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 686 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 687 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 688 } 689 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 690 .addUse(I.getOperand(0).getReg()) 691 .addUse(SrcReg) 692 .addUse(I.getOperand(2).getReg()); 693 if (I.getNumMemOperands()) 694 addMemoryOperands(*I.memoperands_begin(), MIB); 695 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 696 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 697 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 698 .addUse(MIB->getOperand(0).getReg()); 699 return Result; 700 } 701 702 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 703 const SPIRVType *ResType, 704 MachineInstr &I, 705 unsigned NewOpcode, 706 unsigned NegateOpcode) const { 707 assert(I.hasOneMemOperand()); 708 const MachineMemOperand *MemOp = *I.memoperands_begin(); 709 uint32_t Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID())); 710 Register ScopeReg = buildI32Constant(Scope, I); 711 712 Register Ptr = I.getOperand(1).getReg(); 713 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 714 // auto ScSem = 715 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 716 AtomicOrdering AO = MemOp->getSuccessOrdering(); 717 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 718 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 719 720 bool Result = false; 721 Register ValueReg = I.getOperand(2).getReg(); 722 if (NegateOpcode != 0) { 723 // Translation with negative value operand is requested 724 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 725 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 726 ValueReg = TmpReg; 727 } 728 729 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 730 .addDef(ResVReg) 731 .addUse(GR.getSPIRVTypeID(ResType)) 732 .addUse(Ptr) 733 .addUse(ScopeReg) 734 .addUse(MemSemReg) 735 .addUse(ValueReg) 736 .constrainAllUses(TII, TRI, RBI); 737 return Result; 738 } 739 740 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 741 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 742 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 743 Register MemSemReg = buildI32Constant(MemSem, I); 744 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 745 uint32_t Scope = static_cast<uint32_t>(getScope(Ord)); 746 Register ScopeReg = buildI32Constant(Scope, I); 747 MachineBasicBlock &BB = *I.getParent(); 748 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 749 .addUse(ScopeReg) 750 .addUse(MemSemReg) 751 .constrainAllUses(TII, TRI, RBI); 752 } 753 754 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 755 const SPIRVType *ResType, 756 MachineInstr &I) const { 757 Register ScopeReg; 758 Register MemSemEqReg; 759 Register MemSemNeqReg; 760 Register Ptr = I.getOperand(2).getReg(); 761 if (!isa<GIntrinsic>(I)) { 762 assert(I.hasOneMemOperand()); 763 const MachineMemOperand *MemOp = *I.memoperands_begin(); 764 unsigned Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID())); 765 ScopeReg = buildI32Constant(Scope, I); 766 767 unsigned ScSem = static_cast<uint32_t>( 768 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 769 AtomicOrdering AO = MemOp->getSuccessOrdering(); 770 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 771 MemSemEqReg = buildI32Constant(MemSemEq, I); 772 AtomicOrdering FO = MemOp->getFailureOrdering(); 773 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 774 MemSemNeqReg = 775 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 776 } else { 777 ScopeReg = I.getOperand(5).getReg(); 778 MemSemEqReg = I.getOperand(6).getReg(); 779 MemSemNeqReg = I.getOperand(7).getReg(); 780 } 781 782 Register Cmp = I.getOperand(3).getReg(); 783 Register Val = I.getOperand(4).getReg(); 784 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 785 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); 786 const DebugLoc &DL = I.getDebugLoc(); 787 bool Result = 788 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 789 .addDef(ACmpRes) 790 .addUse(GR.getSPIRVTypeID(SpvValTy)) 791 .addUse(Ptr) 792 .addUse(ScopeReg) 793 .addUse(MemSemEqReg) 794 .addUse(MemSemNeqReg) 795 .addUse(Val) 796 .addUse(Cmp) 797 .constrainAllUses(TII, TRI, RBI); 798 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 799 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 800 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 801 .addDef(CmpSuccReg) 802 .addUse(GR.getSPIRVTypeID(BoolTy)) 803 .addUse(ACmpRes) 804 .addUse(Cmp) 805 .constrainAllUses(TII, TRI, RBI); 806 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 807 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 808 .addDef(TmpReg) 809 .addUse(GR.getSPIRVTypeID(ResType)) 810 .addUse(ACmpRes) 811 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 812 .addImm(0) 813 .constrainAllUses(TII, TRI, RBI); 814 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 815 .addDef(ResVReg) 816 .addUse(GR.getSPIRVTypeID(ResType)) 817 .addUse(CmpSuccReg) 818 .addUse(TmpReg) 819 .addImm(1) 820 .constrainAllUses(TII, TRI, RBI); 821 return Result; 822 } 823 824 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 825 switch (SC) { 826 case SPIRV::StorageClass::Workgroup: 827 case SPIRV::StorageClass::CrossWorkgroup: 828 case SPIRV::StorageClass::Function: 829 return true; 830 default: 831 return false; 832 } 833 } 834 835 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 836 switch (SC) { 837 case SPIRV::StorageClass::DeviceOnlyINTEL: 838 case SPIRV::StorageClass::HostOnlyINTEL: 839 return true; 840 default: 841 return false; 842 } 843 } 844 845 // In SPIR-V address space casting can only happen to and from the Generic 846 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 847 // pointers to and from Generic pointers. As such, we can convert e.g. from 848 // Workgroup to Function by going via a Generic pointer as an intermediary. All 849 // other combinations can only be done by a bitcast, and are probably not safe. 850 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 851 const SPIRVType *ResType, 852 MachineInstr &I) const { 853 // If the AddrSpaceCast user is single and in OpConstantComposite or 854 // OpVariable, we should select OpSpecConstantOp. 855 auto UIs = MRI->use_instructions(ResVReg); 856 if (!UIs.empty() && ++UIs.begin() == UIs.end() && 857 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || 858 UIs.begin()->getOpcode() == SPIRV::OpVariable || 859 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { 860 Register NewReg = I.getOperand(1).getReg(); 861 MachineBasicBlock &BB = *I.getParent(); 862 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 863 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, 864 SPIRV::StorageClass::Generic); 865 bool Result = 866 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 867 .addDef(ResVReg) 868 .addUse(GR.getSPIRVTypeID(ResType)) 869 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) 870 .addUse(NewReg) 871 .constrainAllUses(TII, TRI, RBI); 872 return Result; 873 } 874 Register SrcPtr = I.getOperand(1).getReg(); 875 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 876 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); 877 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); 878 879 // don't generate a cast between identical storage classes 880 if (SrcSC == DstSC) 881 return true; 882 883 // Casting from an eligible pointer to Generic. 884 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 885 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 886 // Casting from Generic to an eligible pointer. 887 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 888 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 889 // Casting between 2 eligible pointers using Generic as an intermediary. 890 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 891 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); 892 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 893 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); 894 MachineBasicBlock &BB = *I.getParent(); 895 const DebugLoc &DL = I.getDebugLoc(); 896 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 897 .addDef(Tmp) 898 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 899 .addUse(SrcPtr) 900 .constrainAllUses(TII, TRI, RBI); 901 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 902 .addDef(ResVReg) 903 .addUse(GR.getSPIRVTypeID(ResType)) 904 .addUse(Tmp) 905 .constrainAllUses(TII, TRI, RBI); 906 } 907 908 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 909 // be applied 910 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 911 return selectUnOp(ResVReg, ResType, I, 912 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 913 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 914 return selectUnOp(ResVReg, ResType, I, 915 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 916 917 // TODO Should this case just be disallowed completely? 918 // We're casting 2 other arbitrary address spaces, so have to bitcast. 919 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 920 } 921 922 static unsigned getFCmpOpcode(unsigned PredNum) { 923 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 924 switch (Pred) { 925 case CmpInst::FCMP_OEQ: 926 return SPIRV::OpFOrdEqual; 927 case CmpInst::FCMP_OGE: 928 return SPIRV::OpFOrdGreaterThanEqual; 929 case CmpInst::FCMP_OGT: 930 return SPIRV::OpFOrdGreaterThan; 931 case CmpInst::FCMP_OLE: 932 return SPIRV::OpFOrdLessThanEqual; 933 case CmpInst::FCMP_OLT: 934 return SPIRV::OpFOrdLessThan; 935 case CmpInst::FCMP_ONE: 936 return SPIRV::OpFOrdNotEqual; 937 case CmpInst::FCMP_ORD: 938 return SPIRV::OpOrdered; 939 case CmpInst::FCMP_UEQ: 940 return SPIRV::OpFUnordEqual; 941 case CmpInst::FCMP_UGE: 942 return SPIRV::OpFUnordGreaterThanEqual; 943 case CmpInst::FCMP_UGT: 944 return SPIRV::OpFUnordGreaterThan; 945 case CmpInst::FCMP_ULE: 946 return SPIRV::OpFUnordLessThanEqual; 947 case CmpInst::FCMP_ULT: 948 return SPIRV::OpFUnordLessThan; 949 case CmpInst::FCMP_UNE: 950 return SPIRV::OpFUnordNotEqual; 951 case CmpInst::FCMP_UNO: 952 return SPIRV::OpUnordered; 953 default: 954 llvm_unreachable("Unknown predicate type for FCmp"); 955 } 956 } 957 958 static unsigned getICmpOpcode(unsigned PredNum) { 959 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 960 switch (Pred) { 961 case CmpInst::ICMP_EQ: 962 return SPIRV::OpIEqual; 963 case CmpInst::ICMP_NE: 964 return SPIRV::OpINotEqual; 965 case CmpInst::ICMP_SGE: 966 return SPIRV::OpSGreaterThanEqual; 967 case CmpInst::ICMP_SGT: 968 return SPIRV::OpSGreaterThan; 969 case CmpInst::ICMP_SLE: 970 return SPIRV::OpSLessThanEqual; 971 case CmpInst::ICMP_SLT: 972 return SPIRV::OpSLessThan; 973 case CmpInst::ICMP_UGE: 974 return SPIRV::OpUGreaterThanEqual; 975 case CmpInst::ICMP_UGT: 976 return SPIRV::OpUGreaterThan; 977 case CmpInst::ICMP_ULE: 978 return SPIRV::OpULessThanEqual; 979 case CmpInst::ICMP_ULT: 980 return SPIRV::OpULessThan; 981 default: 982 llvm_unreachable("Unknown predicate type for ICmp"); 983 } 984 } 985 986 static unsigned getPtrCmpOpcode(unsigned Pred) { 987 switch (static_cast<CmpInst::Predicate>(Pred)) { 988 case CmpInst::ICMP_EQ: 989 return SPIRV::OpPtrEqual; 990 case CmpInst::ICMP_NE: 991 return SPIRV::OpPtrNotEqual; 992 default: 993 llvm_unreachable("Unknown predicate type for pointer comparison"); 994 } 995 } 996 997 // Return the logical operation, or abort if none exists. 998 static unsigned getBoolCmpOpcode(unsigned PredNum) { 999 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1000 switch (Pred) { 1001 case CmpInst::ICMP_EQ: 1002 return SPIRV::OpLogicalEqual; 1003 case CmpInst::ICMP_NE: 1004 return SPIRV::OpLogicalNotEqual; 1005 default: 1006 llvm_unreachable("Unknown predicate type for Bool comparison"); 1007 } 1008 } 1009 1010 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1011 const SPIRVType *ResType, 1012 MachineInstr &I) const { 1013 MachineBasicBlock &BB = *I.getParent(); 1014 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1015 .addDef(ResVReg) 1016 .addUse(GR.getSPIRVTypeID(ResType)) 1017 .addUse(I.getOperand(1).getReg()) 1018 .constrainAllUses(TII, TRI, RBI); 1019 } 1020 1021 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1022 const SPIRVType *ResType, 1023 MachineInstr &I) const { 1024 // There is no way to implement `freeze` correctly without support on SPIR-V 1025 // standard side, but we may at least address a simple (static) case when 1026 // undef/poison value presence is obvious. The main benefit of even 1027 // incomplete `freeze` support is preventing of translation from crashing due 1028 // to lack of support on legalization and instruction selection steps. 1029 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1030 return false; 1031 Register OpReg = I.getOperand(1).getReg(); 1032 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1033 Register Reg; 1034 switch (Def->getOpcode()) { 1035 case SPIRV::ASSIGN_TYPE: 1036 if (MachineInstr *AssignToDef = 1037 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1038 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1039 Reg = Def->getOperand(2).getReg(); 1040 } 1041 break; 1042 case SPIRV::OpUndef: 1043 Reg = Def->getOperand(1).getReg(); 1044 break; 1045 } 1046 unsigned DestOpCode; 1047 if (Reg.isValid()) { 1048 DestOpCode = SPIRV::OpConstantNull; 1049 } else { 1050 DestOpCode = TargetOpcode::COPY; 1051 Reg = OpReg; 1052 } 1053 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1054 .addDef(I.getOperand(0).getReg()) 1055 .addUse(Reg) 1056 .constrainAllUses(TII, TRI, RBI); 1057 } 1058 return false; 1059 } 1060 1061 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, 1062 const SPIRVType *ResType, 1063 MachineInstr &I) const { 1064 // TODO: only const case is supported for now. 1065 assert(std::all_of( 1066 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { 1067 if (MO.isDef()) 1068 return true; 1069 if (!MO.isReg()) 1070 return false; 1071 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); 1072 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 1073 ConstTy->getOperand(1).isReg()); 1074 Register ConstReg = ConstTy->getOperand(1).getReg(); 1075 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 1076 assert(Const); 1077 return (Const->getOpcode() == TargetOpcode::G_CONSTANT || 1078 Const->getOpcode() == TargetOpcode::G_FCONSTANT); 1079 })); 1080 1081 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1082 TII.get(SPIRV::OpConstantComposite)) 1083 .addDef(ResVReg) 1084 .addUse(GR.getSPIRVTypeID(ResType)); 1085 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 1086 MIB.addUse(I.getOperand(i).getReg()); 1087 return MIB.constrainAllUses(TII, TRI, RBI); 1088 } 1089 1090 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 1091 const SPIRVType *ResType, 1092 unsigned CmpOpc, 1093 MachineInstr &I) const { 1094 Register Cmp0 = I.getOperand(2).getReg(); 1095 Register Cmp1 = I.getOperand(3).getReg(); 1096 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1097 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1098 "CMP operands should have the same type"); 1099 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1100 .addDef(ResVReg) 1101 .addUse(GR.getSPIRVTypeID(ResType)) 1102 .addUse(Cmp0) 1103 .addUse(Cmp1) 1104 .constrainAllUses(TII, TRI, RBI); 1105 } 1106 1107 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1108 const SPIRVType *ResType, 1109 MachineInstr &I) const { 1110 auto Pred = I.getOperand(1).getPredicate(); 1111 unsigned CmpOpc; 1112 1113 Register CmpOperand = I.getOperand(2).getReg(); 1114 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1115 CmpOpc = getPtrCmpOpcode(Pred); 1116 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1117 CmpOpc = getBoolCmpOpcode(Pred); 1118 else 1119 CmpOpc = getICmpOpcode(Pred); 1120 return selectCmp(ResVReg, ResType, CmpOpc, I); 1121 } 1122 1123 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, 1124 const MachineInstr &I, 1125 int OpIdx) const { 1126 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1127 "Expected G_FCONSTANT"); 1128 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1129 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1130 } 1131 1132 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1133 const MachineInstr &I, 1134 int OpIdx) const { 1135 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1136 "Expected G_CONSTANT"); 1137 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1138 } 1139 1140 Register 1141 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1142 const SPIRVType *ResType) const { 1143 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1144 const SPIRVType *SpvI32Ty = 1145 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1146 // Find a constant in DT or build a new one. 1147 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1148 Register NewReg = GR.find(ConstInt, GR.CurMF); 1149 if (!NewReg.isValid()) { 1150 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1151 GR.add(ConstInt, GR.CurMF, NewReg); 1152 MachineInstr *MI; 1153 MachineBasicBlock &BB = *I.getParent(); 1154 if (Val == 0) { 1155 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1156 .addDef(NewReg) 1157 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 1158 } else { 1159 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1160 .addDef(NewReg) 1161 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 1162 .addImm(APInt(32, Val).getZExtValue()); 1163 } 1164 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 1165 } 1166 return NewReg; 1167 } 1168 1169 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 1170 const SPIRVType *ResType, 1171 MachineInstr &I) const { 1172 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 1173 return selectCmp(ResVReg, ResType, CmpOp, I); 1174 } 1175 1176 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 1177 MachineInstr &I) const { 1178 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1179 return GR.getOrCreateConsIntVector(0, I, ResType, TII); 1180 return GR.getOrCreateConstInt(0, I, ResType, TII); 1181 } 1182 1183 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 1184 const SPIRVType *ResType, 1185 MachineInstr &I) const { 1186 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1187 APInt One = 1188 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 1189 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1190 return GR.getOrCreateConsIntVector(One.getZExtValue(), I, ResType, TII); 1191 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 1192 } 1193 1194 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 1195 const SPIRVType *ResType, 1196 MachineInstr &I, 1197 bool IsSigned) const { 1198 // To extend a bool, we need to use OpSelect between constants. 1199 Register ZeroReg = buildZerosVal(ResType, I); 1200 Register OneReg = buildOnesVal(IsSigned, ResType, I); 1201 bool IsScalarBool = 1202 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 1203 unsigned Opcode = 1204 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 1205 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1206 .addDef(ResVReg) 1207 .addUse(GR.getSPIRVTypeID(ResType)) 1208 .addUse(I.getOperand(1).getReg()) 1209 .addUse(OneReg) 1210 .addUse(ZeroReg) 1211 .constrainAllUses(TII, TRI, RBI); 1212 } 1213 1214 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 1215 const SPIRVType *ResType, 1216 MachineInstr &I, bool IsSigned, 1217 unsigned Opcode) const { 1218 Register SrcReg = I.getOperand(1).getReg(); 1219 // We can convert bool value directly to float type without OpConvert*ToF, 1220 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 1221 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 1222 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1223 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 1224 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 1225 const unsigned NumElts = ResType->getOperand(2).getImm(); 1226 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 1227 } 1228 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1229 selectSelect(SrcReg, TmpType, I, false); 1230 } 1231 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 1232 } 1233 1234 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 1235 const SPIRVType *ResType, 1236 MachineInstr &I, bool IsSigned) const { 1237 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) 1238 return selectSelect(ResVReg, ResType, I, IsSigned); 1239 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1240 return selectUnOp(ResVReg, ResType, I, Opcode); 1241 } 1242 1243 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 1244 Register ResVReg, 1245 MachineInstr &I, 1246 const SPIRVType *IntTy, 1247 const SPIRVType *BoolTy) const { 1248 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 1249 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1250 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 1251 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 1252 Register Zero = buildZerosVal(IntTy, I); 1253 Register One = buildOnesVal(false, IntTy, I); 1254 MachineBasicBlock &BB = *I.getParent(); 1255 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1256 .addDef(BitIntReg) 1257 .addUse(GR.getSPIRVTypeID(IntTy)) 1258 .addUse(IntReg) 1259 .addUse(One) 1260 .constrainAllUses(TII, TRI, RBI); 1261 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1262 .addDef(ResVReg) 1263 .addUse(GR.getSPIRVTypeID(BoolTy)) 1264 .addUse(BitIntReg) 1265 .addUse(Zero) 1266 .constrainAllUses(TII, TRI, RBI); 1267 } 1268 1269 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 1270 const SPIRVType *ResType, 1271 MachineInstr &I) const { 1272 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { 1273 Register IntReg = I.getOperand(1).getReg(); 1274 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 1275 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 1276 } 1277 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 1278 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1279 return selectUnOp(ResVReg, ResType, I, Opcode); 1280 } 1281 1282 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 1283 const SPIRVType *ResType, 1284 const APInt &Imm, 1285 MachineInstr &I) const { 1286 unsigned TyOpcode = ResType->getOpcode(); 1287 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 1288 MachineBasicBlock &BB = *I.getParent(); 1289 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 1290 Imm.isZero()) 1291 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1292 .addDef(ResVReg) 1293 .addUse(GR.getSPIRVTypeID(ResType)) 1294 .constrainAllUses(TII, TRI, RBI); 1295 if (TyOpcode == SPIRV::OpTypeInt) { 1296 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 1297 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 1298 if (Reg == ResVReg) 1299 return true; 1300 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1301 .addDef(ResVReg) 1302 .addUse(Reg) 1303 .constrainAllUses(TII, TRI, RBI); 1304 } 1305 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1306 .addDef(ResVReg) 1307 .addUse(GR.getSPIRVTypeID(ResType)); 1308 // <=32-bit integers should be caught by the sdag pattern. 1309 assert(Imm.getBitWidth() > 32); 1310 addNumImm(Imm, MIB); 1311 return MIB.constrainAllUses(TII, TRI, RBI); 1312 } 1313 1314 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 1315 const SPIRVType *ResType, 1316 MachineInstr &I) const { 1317 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1318 .addDef(ResVReg) 1319 .addUse(GR.getSPIRVTypeID(ResType)) 1320 .constrainAllUses(TII, TRI, RBI); 1321 } 1322 1323 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1324 assert(MO.isReg()); 1325 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1326 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) 1327 return false; 1328 assert(TypeInst->getOperand(1).isReg()); 1329 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1330 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 1331 } 1332 1333 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1334 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1335 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1336 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 1337 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 1338 } 1339 1340 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 1341 const SPIRVType *ResType, 1342 MachineInstr &I) const { 1343 MachineBasicBlock &BB = *I.getParent(); 1344 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 1345 .addDef(ResVReg) 1346 .addUse(GR.getSPIRVTypeID(ResType)) 1347 // object to insert 1348 .addUse(I.getOperand(3).getReg()) 1349 // composite to insert into 1350 .addUse(I.getOperand(2).getReg()); 1351 for (unsigned i = 4; i < I.getNumOperands(); i++) 1352 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1353 return MIB.constrainAllUses(TII, TRI, RBI); 1354 } 1355 1356 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 1357 const SPIRVType *ResType, 1358 MachineInstr &I) const { 1359 MachineBasicBlock &BB = *I.getParent(); 1360 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1361 .addDef(ResVReg) 1362 .addUse(GR.getSPIRVTypeID(ResType)) 1363 .addUse(I.getOperand(2).getReg()); 1364 for (unsigned i = 3; i < I.getNumOperands(); i++) 1365 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1366 return MIB.constrainAllUses(TII, TRI, RBI); 1367 } 1368 1369 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 1370 const SPIRVType *ResType, 1371 MachineInstr &I) const { 1372 if (isImm(I.getOperand(4), MRI)) 1373 return selectInsertVal(ResVReg, ResType, I); 1374 MachineBasicBlock &BB = *I.getParent(); 1375 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 1376 .addDef(ResVReg) 1377 .addUse(GR.getSPIRVTypeID(ResType)) 1378 .addUse(I.getOperand(2).getReg()) 1379 .addUse(I.getOperand(3).getReg()) 1380 .addUse(I.getOperand(4).getReg()) 1381 .constrainAllUses(TII, TRI, RBI); 1382 } 1383 1384 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 1385 const SPIRVType *ResType, 1386 MachineInstr &I) const { 1387 if (isImm(I.getOperand(3), MRI)) 1388 return selectExtractVal(ResVReg, ResType, I); 1389 MachineBasicBlock &BB = *I.getParent(); 1390 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 1391 .addDef(ResVReg) 1392 .addUse(GR.getSPIRVTypeID(ResType)) 1393 .addUse(I.getOperand(2).getReg()) 1394 .addUse(I.getOperand(3).getReg()) 1395 .constrainAllUses(TII, TRI, RBI); 1396 } 1397 1398 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 1399 const SPIRVType *ResType, 1400 MachineInstr &I) const { 1401 const bool IsGEPInBounds = I.getOperand(2).getImm(); 1402 1403 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 1404 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 1405 // we have to use Op[InBounds]AccessChain. 1406 const unsigned Opcode = STI.isVulkanEnv() 1407 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 1408 : SPIRV::OpAccessChain) 1409 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 1410 : SPIRV::OpPtrAccessChain); 1411 1412 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1413 .addDef(ResVReg) 1414 .addUse(GR.getSPIRVTypeID(ResType)) 1415 // Object to get a pointer to. 1416 .addUse(I.getOperand(3).getReg()); 1417 // Adding indices. 1418 const unsigned StartingIndex = 1419 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 1420 ? 5 1421 : 4; 1422 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 1423 Res.addUse(I.getOperand(i).getReg()); 1424 return Res.constrainAllUses(TII, TRI, RBI); 1425 } 1426 1427 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 1428 const SPIRVType *ResType, 1429 MachineInstr &I) const { 1430 MachineBasicBlock &BB = *I.getParent(); 1431 switch (cast<GIntrinsic>(I).getIntrinsicID()) { 1432 case Intrinsic::spv_load: 1433 return selectLoad(ResVReg, ResType, I); 1434 case Intrinsic::spv_store: 1435 return selectStore(I); 1436 case Intrinsic::spv_extractv: 1437 return selectExtractVal(ResVReg, ResType, I); 1438 case Intrinsic::spv_insertv: 1439 return selectInsertVal(ResVReg, ResType, I); 1440 case Intrinsic::spv_extractelt: 1441 return selectExtractElt(ResVReg, ResType, I); 1442 case Intrinsic::spv_insertelt: 1443 return selectInsertElt(ResVReg, ResType, I); 1444 case Intrinsic::spv_gep: 1445 return selectGEP(ResVReg, ResType, I); 1446 case Intrinsic::spv_unref_global: 1447 case Intrinsic::spv_init_global: { 1448 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 1449 MachineInstr *Init = I.getNumExplicitOperands() > 2 1450 ? MRI->getVRegDef(I.getOperand(2).getReg()) 1451 : nullptr; 1452 assert(MI); 1453 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 1454 } 1455 case Intrinsic::spv_undef: { 1456 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1457 .addDef(ResVReg) 1458 .addUse(GR.getSPIRVTypeID(ResType)); 1459 return MIB.constrainAllUses(TII, TRI, RBI); 1460 } 1461 case Intrinsic::spv_const_composite: { 1462 // If no values are attached, the composite is null constant. 1463 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 1464 unsigned Opcode = 1465 IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite; 1466 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1467 .addDef(ResVReg) 1468 .addUse(GR.getSPIRVTypeID(ResType)); 1469 // skip type MD node we already used when generated assign.type for this 1470 if (!IsNull) { 1471 for (unsigned i = I.getNumExplicitDefs() + 1; 1472 i < I.getNumExplicitOperands(); ++i) { 1473 MIB.addUse(I.getOperand(i).getReg()); 1474 } 1475 } 1476 return MIB.constrainAllUses(TII, TRI, RBI); 1477 } 1478 case Intrinsic::spv_assign_name: { 1479 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 1480 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 1481 for (unsigned i = I.getNumExplicitDefs() + 2; 1482 i < I.getNumExplicitOperands(); ++i) { 1483 MIB.addImm(I.getOperand(i).getImm()); 1484 } 1485 return MIB.constrainAllUses(TII, TRI, RBI); 1486 } 1487 case Intrinsic::spv_switch: { 1488 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 1489 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 1490 if (I.getOperand(i).isReg()) 1491 MIB.addReg(I.getOperand(i).getReg()); 1492 else if (I.getOperand(i).isCImm()) 1493 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 1494 else if (I.getOperand(i).isMBB()) 1495 MIB.addMBB(I.getOperand(i).getMBB()); 1496 else 1497 llvm_unreachable("Unexpected OpSwitch operand"); 1498 } 1499 return MIB.constrainAllUses(TII, TRI, RBI); 1500 } 1501 case Intrinsic::spv_cmpxchg: 1502 return selectAtomicCmpXchg(ResVReg, ResType, I); 1503 case Intrinsic::spv_unreachable: 1504 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 1505 break; 1506 case Intrinsic::spv_alloca: 1507 return selectFrameIndex(ResVReg, ResType, I); 1508 case Intrinsic::spv_assume: 1509 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1510 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 1511 .addUse(I.getOperand(1).getReg()); 1512 break; 1513 case Intrinsic::spv_expect: 1514 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1515 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 1516 .addDef(ResVReg) 1517 .addUse(GR.getSPIRVTypeID(ResType)) 1518 .addUse(I.getOperand(2).getReg()) 1519 .addUse(I.getOperand(3).getReg()); 1520 break; 1521 default: 1522 llvm_unreachable("Intrinsic selection not implemented"); 1523 } 1524 return true; 1525 } 1526 1527 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 1528 const SPIRVType *ResType, 1529 MachineInstr &I) const { 1530 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 1531 .addDef(ResVReg) 1532 .addUse(GR.getSPIRVTypeID(ResType)) 1533 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 1534 .constrainAllUses(TII, TRI, RBI); 1535 } 1536 1537 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 1538 // InstructionSelector walks backwards through the instructions. We can use 1539 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 1540 // first, so can generate an OpBranchConditional here. If there is no 1541 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 1542 const MachineInstr *PrevI = I.getPrevNode(); 1543 MachineBasicBlock &MBB = *I.getParent(); 1544 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 1545 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1546 .addUse(PrevI->getOperand(0).getReg()) 1547 .addMBB(PrevI->getOperand(1).getMBB()) 1548 .addMBB(I.getOperand(0).getMBB()) 1549 .constrainAllUses(TII, TRI, RBI); 1550 } 1551 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 1552 .addMBB(I.getOperand(0).getMBB()) 1553 .constrainAllUses(TII, TRI, RBI); 1554 } 1555 1556 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 1557 // InstructionSelector walks backwards through the instructions. For an 1558 // explicit conditional branch with no fallthrough, we use both a G_BR and a 1559 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 1560 // generate the OpBranchConditional in selectBranch above. 1561 // 1562 // If an OpBranchConditional has been generated, we simply return, as the work 1563 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 1564 // implicit fallthrough to the next basic block, so we need to create an 1565 // OpBranchConditional with an explicit "false" argument pointing to the next 1566 // basic block that LLVM would fall through to. 1567 const MachineInstr *NextI = I.getNextNode(); 1568 // Check if this has already been successfully selected. 1569 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 1570 return true; 1571 // Must be relying on implicit block fallthrough, so generate an 1572 // OpBranchConditional with the "next" basic block as the "false" target. 1573 MachineBasicBlock &MBB = *I.getParent(); 1574 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 1575 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 1576 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1577 .addUse(I.getOperand(0).getReg()) 1578 .addMBB(I.getOperand(1).getMBB()) 1579 .addMBB(NextMBB) 1580 .constrainAllUses(TII, TRI, RBI); 1581 } 1582 1583 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 1584 const SPIRVType *ResType, 1585 MachineInstr &I) const { 1586 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 1587 .addDef(ResVReg) 1588 .addUse(GR.getSPIRVTypeID(ResType)); 1589 const unsigned NumOps = I.getNumOperands(); 1590 for (unsigned i = 1; i < NumOps; i += 2) { 1591 MIB.addUse(I.getOperand(i + 0).getReg()); 1592 MIB.addMBB(I.getOperand(i + 1).getMBB()); 1593 } 1594 return MIB.constrainAllUses(TII, TRI, RBI); 1595 } 1596 1597 bool SPIRVInstructionSelector::selectGlobalValue( 1598 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 1599 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 1600 MachineIRBuilder MIRBuilder(I); 1601 const GlobalValue *GV = I.getOperand(1).getGlobal(); 1602 Type *GVType = GV->getValueType(); 1603 SPIRVType *PointerBaseType; 1604 if (GVType->isArrayTy()) { 1605 SPIRVType *ArrayElementType = 1606 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 1607 SPIRV::AccessQualifier::ReadWrite, false); 1608 PointerBaseType = GR.getOrCreateSPIRVArrayType( 1609 ArrayElementType, GVType->getArrayNumElements(), I, TII); 1610 } else { 1611 PointerBaseType = GR.getOrCreateSPIRVType( 1612 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1613 } 1614 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 1615 PointerBaseType, I, TII, 1616 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 1617 1618 std::string GlobalIdent; 1619 if (!GV->hasName()) { 1620 unsigned &ID = UnnamedGlobalIDs[GV]; 1621 if (ID == 0) 1622 ID = UnnamedGlobalIDs.size(); 1623 GlobalIdent = "__unnamed_" + Twine(ID).str(); 1624 } else { 1625 GlobalIdent = GV->getGlobalIdentifier(); 1626 } 1627 1628 // Behaviour of functions as operands depends on availability of the 1629 // corresponding extension (SPV_INTEL_function_pointers): 1630 // - If there is an extension to operate with functions as operands: 1631 // We create a proper constant operand and evaluate a correct type for a 1632 // function pointer. 1633 // - Without the required extension: 1634 // We have functions as operands in tests with blocks of instruction e.g. in 1635 // transcoding/global_block.ll. These operands are not used and should be 1636 // substituted by zero constants. Their type is expected to be always 1637 // OpTypePointer Function %uchar. 1638 if (isa<Function>(GV)) { 1639 const Constant *ConstVal = GV; 1640 MachineBasicBlock &BB = *I.getParent(); 1641 Register NewReg = GR.find(ConstVal, GR.CurMF); 1642 if (!NewReg.isValid()) { 1643 Register NewReg = ResVReg; 1644 GR.add(ConstVal, GR.CurMF, NewReg); 1645 const Function *GVFun = 1646 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 1647 ? dyn_cast<Function>(GV) 1648 : nullptr; 1649 if (GVFun) { 1650 // References to a function via function pointers generate virtual 1651 // registers without a definition. We will resolve it later, during 1652 // module analysis stage. 1653 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 1654 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1655 MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); 1656 MachineInstrBuilder MB = 1657 BuildMI(BB, I, I.getDebugLoc(), 1658 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 1659 .addDef(NewReg) 1660 .addUse(GR.getSPIRVTypeID(ResType)) 1661 .addUse(FuncVReg); 1662 // mapping the function pointer to the used Function 1663 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 1664 return MB.constrainAllUses(TII, TRI, RBI); 1665 } 1666 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1667 .addDef(NewReg) 1668 .addUse(GR.getSPIRVTypeID(ResType)) 1669 .constrainAllUses(TII, TRI, RBI); 1670 } 1671 assert(NewReg != ResVReg); 1672 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1673 .addDef(ResVReg) 1674 .addUse(NewReg) 1675 .constrainAllUses(TII, TRI, RBI); 1676 } 1677 auto GlobalVar = cast<GlobalVariable>(GV); 1678 assert(GlobalVar->getName() != "llvm.global.annotations"); 1679 1680 bool HasInit = GlobalVar->hasInitializer() && 1681 !isa<UndefValue>(GlobalVar->getInitializer()); 1682 // Skip empty declaration for GVs with initilaizers till we get the decl with 1683 // passed initializer. 1684 if (HasInit && !Init) 1685 return true; 1686 1687 unsigned AddrSpace = GV->getAddressSpace(); 1688 SPIRV::StorageClass::StorageClass Storage = 1689 addressSpaceToStorageClass(AddrSpace, STI); 1690 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 1691 Storage != SPIRV::StorageClass::Function; 1692 SPIRV::LinkageType::LinkageType LnkType = 1693 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 1694 ? SPIRV::LinkageType::Import 1695 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 1696 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 1697 ? SPIRV::LinkageType::LinkOnceODR 1698 : SPIRV::LinkageType::Export); 1699 1700 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 1701 Storage, Init, GlobalVar->isConstant(), 1702 HasLnkTy, LnkType, MIRBuilder, true); 1703 return Reg.isValid(); 1704 } 1705 1706 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 1707 const SPIRVType *ResType, 1708 MachineInstr &I) const { 1709 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 1710 return selectExtInst(ResVReg, ResType, I, CL::log10); 1711 } 1712 1713 // There is no log10 instruction in the GLSL Extended Instruction set, so it 1714 // is implemented as: 1715 // log10(x) = log2(x) * (1 / log2(10)) 1716 // = log2(x) * 0.30103 1717 1718 MachineIRBuilder MIRBuilder(I); 1719 MachineBasicBlock &BB = *I.getParent(); 1720 1721 // Build log2(x). 1722 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1723 bool Result = 1724 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 1725 .addDef(VarReg) 1726 .addUse(GR.getSPIRVTypeID(ResType)) 1727 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 1728 .addImm(GL::Log2) 1729 .add(I.getOperand(1)) 1730 .constrainAllUses(TII, TRI, RBI); 1731 1732 // Build 0.30103. 1733 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 1734 ResType->getOpcode() == SPIRV::OpTypeFloat); 1735 // TODO: Add matrix implementation once supported by the HLSL frontend. 1736 const SPIRVType *SpirvScalarType = 1737 ResType->getOpcode() == SPIRV::OpTypeVector 1738 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 1739 : ResType; 1740 Register ScaleReg = 1741 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 1742 1743 // Multiply log2(x) by 0.30103 to get log10(x) result. 1744 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 1745 ? SPIRV::OpVectorTimesScalar 1746 : SPIRV::OpFMulS; 1747 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1748 .addDef(ResVReg) 1749 .addUse(GR.getSPIRVTypeID(ResType)) 1750 .addUse(VarReg) 1751 .addUse(ScaleReg) 1752 .constrainAllUses(TII, TRI, RBI); 1753 1754 return Result; 1755 } 1756 1757 namespace llvm { 1758 InstructionSelector * 1759 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 1760 const SPIRVSubtarget &Subtarget, 1761 const RegisterBankInfo &RBI) { 1762 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 1763 } 1764 } // namespace llvm 1765