1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the targeting of the InstructionSelector class for 10 // SPIRV. 11 // TODO: This should be generated by TableGen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/SPIRVBaseInfo.h" 16 #include "MCTargetDesc/SPIRVMCTargetDesc.h" 17 #include "SPIRV.h" 18 #include "SPIRVGlobalRegistry.h" 19 #include "SPIRVInstrInfo.h" 20 #include "SPIRVRegisterBankInfo.h" 21 #include "SPIRVRegisterInfo.h" 22 #include "SPIRVTargetMachine.h" 23 #include "SPIRVUtils.h" 24 #include "llvm/ADT/APFloat.h" 25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" 26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 28 #include "llvm/CodeGen/MachineInstrBuilder.h" 29 #include "llvm/CodeGen/MachineModuleInfoImpls.h" 30 #include "llvm/CodeGen/MachineRegisterInfo.h" 31 #include "llvm/IR/IntrinsicsSPIRV.h" 32 #include "llvm/Support/Debug.h" 33 34 namespace llvm { 35 36 class SPIRVMachineModuleInfo : public MachineModuleInfoImpl { 37 public: 38 SyncScope::ID Work_ItemSSID; 39 SyncScope::ID WorkGroupSSID; 40 SyncScope::ID DeviceSSID; 41 SyncScope::ID AllSVMDevicesSSID; 42 SyncScope::ID SubGroupSSID; 43 44 SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) { 45 LLVMContext &CTX = MMI.getModule()->getContext(); 46 Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item"); 47 WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); 48 DeviceSSID = CTX.getOrInsertSyncScopeID("device"); 49 AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices"); 50 SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group"); 51 } 52 }; 53 54 } // end namespace llvm 55 56 #define DEBUG_TYPE "spirv-isel" 57 58 using namespace llvm; 59 namespace CL = SPIRV::OpenCLExtInst; 60 namespace GL = SPIRV::GLSLExtInst; 61 62 using ExtInstList = 63 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; 64 65 namespace { 66 67 #define GET_GLOBALISEL_PREDICATE_BITSET 68 #include "SPIRVGenGlobalISel.inc" 69 #undef GET_GLOBALISEL_PREDICATE_BITSET 70 71 class SPIRVInstructionSelector : public InstructionSelector { 72 const SPIRVSubtarget &STI; 73 const SPIRVInstrInfo &TII; 74 const SPIRVRegisterInfo &TRI; 75 const RegisterBankInfo &RBI; 76 SPIRVGlobalRegistry &GR; 77 MachineRegisterInfo *MRI; 78 SPIRVMachineModuleInfo *MMI = nullptr; 79 80 /// We need to keep track of the number we give to anonymous global values to 81 /// generate the same name every time when this is needed. 82 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; 83 84 public: 85 SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 86 const SPIRVSubtarget &ST, 87 const RegisterBankInfo &RBI); 88 void setupMF(MachineFunction &MF, GISelKnownBits *KB, 89 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 90 BlockFrequencyInfo *BFI) override; 91 // Common selection code. Instruction-specific selection occurs in spvSelect. 92 bool select(MachineInstr &I) override; 93 static const char *getName() { return DEBUG_TYPE; } 94 95 #define GET_GLOBALISEL_PREDICATES_DECL 96 #include "SPIRVGenGlobalISel.inc" 97 #undef GET_GLOBALISEL_PREDICATES_DECL 98 99 #define GET_GLOBALISEL_TEMPORARIES_DECL 100 #include "SPIRVGenGlobalISel.inc" 101 #undef GET_GLOBALISEL_TEMPORARIES_DECL 102 103 private: 104 // tblgen-erated 'select' implementation, used as the initial selector for 105 // the patterns that don't require complex C++. 106 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 107 108 // All instruction-specific selection that didn't happen in "select()". 109 // Is basically a large Switch/Case delegating to all other select method. 110 bool spvSelect(Register ResVReg, const SPIRVType *ResType, 111 MachineInstr &I) const; 112 113 bool selectGlobalValue(Register ResVReg, MachineInstr &I, 114 const MachineInstr *Init = nullptr) const; 115 116 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, 117 MachineInstr &I, Register SrcReg, 118 unsigned Opcode) const; 119 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 120 unsigned Opcode) const; 121 122 bool selectBitcast(Register ResVReg, const SPIRVType *ResType, 123 MachineInstr &I) const; 124 125 bool selectLoad(Register ResVReg, const SPIRVType *ResType, 126 MachineInstr &I) const; 127 bool selectStore(MachineInstr &I) const; 128 129 bool selectStackSave(Register ResVReg, const SPIRVType *ResType, 130 MachineInstr &I) const; 131 bool selectStackRestore(MachineInstr &I) const; 132 133 bool selectMemOperation(Register ResVReg, MachineInstr &I) const; 134 135 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, 136 MachineInstr &I, unsigned NewOpcode, 137 unsigned NegateOpcode = 0) const; 138 139 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, 140 MachineInstr &I) const; 141 142 bool selectFence(MachineInstr &I) const; 143 144 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, 145 MachineInstr &I) const; 146 147 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, 148 MachineInstr &I) const; 149 150 bool selectConstVector(Register ResVReg, const SPIRVType *ResType, 151 MachineInstr &I) const; 152 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, 153 MachineInstr &I) const; 154 155 bool selectCmp(Register ResVReg, const SPIRVType *ResType, 156 unsigned comparisonOpcode, MachineInstr &I) const; 157 158 bool selectICmp(Register ResVReg, const SPIRVType *ResType, 159 MachineInstr &I) const; 160 bool selectFCmp(Register ResVReg, const SPIRVType *ResType, 161 MachineInstr &I) const; 162 163 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 164 int OpIdx) const; 165 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, 166 int OpIdx) const; 167 168 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, 169 MachineInstr &I) const; 170 171 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 172 bool IsSigned) const; 173 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 174 bool IsSigned, unsigned Opcode) const; 175 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, 176 bool IsSigned) const; 177 178 bool selectTrunc(Register ResVReg, const SPIRVType *ResType, 179 MachineInstr &I) const; 180 181 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, 182 const SPIRVType *intTy, const SPIRVType *boolTy) const; 183 184 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, 185 MachineInstr &I) const; 186 bool selectFreeze(Register ResVReg, const SPIRVType *ResType, 187 MachineInstr &I) const; 188 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, 189 MachineInstr &I) const; 190 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, 191 MachineInstr &I) const; 192 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, 193 MachineInstr &I) const; 194 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, 195 MachineInstr &I) const; 196 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, 197 MachineInstr &I) const; 198 bool selectGEP(Register ResVReg, const SPIRVType *ResType, 199 MachineInstr &I) const; 200 201 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, 202 MachineInstr &I) const; 203 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, 204 MachineInstr &I) const; 205 206 bool selectBranch(MachineInstr &I) const; 207 bool selectBranchCond(MachineInstr &I) const; 208 209 bool selectPhi(Register ResVReg, const SPIRVType *ResType, 210 MachineInstr &I) const; 211 212 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 213 MachineInstr &I, CL::OpenCLExtInst CLInst) const; 214 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 215 MachineInstr &I, CL::OpenCLExtInst CLInst, 216 GL::GLSLExtInst GLInst) const; 217 bool selectExtInst(Register ResVReg, const SPIRVType *ResType, 218 MachineInstr &I, const ExtInstList &ExtInsts) const; 219 220 bool selectLog10(Register ResVReg, const SPIRVType *ResType, 221 MachineInstr &I) const; 222 223 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, 224 MachineInstr &I) const; 225 226 bool selectUnmergeValues(MachineInstr &I) const; 227 228 Register buildI32Constant(uint32_t Val, MachineInstr &I, 229 const SPIRVType *ResType = nullptr) const; 230 231 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; 232 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, 233 MachineInstr &I) const; 234 235 bool wrapIntoSpecConstantOp(MachineInstr &I, 236 SmallVector<Register> &CompositeArgs) const; 237 }; 238 239 } // end anonymous namespace 240 241 #define GET_GLOBALISEL_IMPL 242 #include "SPIRVGenGlobalISel.inc" 243 #undef GET_GLOBALISEL_IMPL 244 245 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, 246 const SPIRVSubtarget &ST, 247 const RegisterBankInfo &RBI) 248 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), 249 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), 250 #define GET_GLOBALISEL_PREDICATES_INIT 251 #include "SPIRVGenGlobalISel.inc" 252 #undef GET_GLOBALISEL_PREDICATES_INIT 253 #define GET_GLOBALISEL_TEMPORARIES_INIT 254 #include "SPIRVGenGlobalISel.inc" 255 #undef GET_GLOBALISEL_TEMPORARIES_INIT 256 { 257 } 258 259 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, 260 CodeGenCoverage *CoverageInfo, 261 ProfileSummaryInfo *PSI, 262 BlockFrequencyInfo *BFI) { 263 MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>(); 264 MRI = &MF.getRegInfo(); 265 GR.setCurrentFunc(MF); 266 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); 267 } 268 269 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); 270 271 // Defined in SPIRVLegalizerInfo.cpp. 272 extern bool isTypeFoldingSupported(unsigned Opcode); 273 274 bool SPIRVInstructionSelector::select(MachineInstr &I) { 275 assert(I.getParent() && "Instruction should be in a basic block!"); 276 assert(I.getParent()->getParent() && "Instruction should be in a function!"); 277 278 Register Opcode = I.getOpcode(); 279 // If it's not a GMIR instruction, we've selected it already. 280 if (!isPreISelGenericOpcode(Opcode)) { 281 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. 282 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg()); 283 if (isTypeFoldingSupported(Def->getOpcode())) { 284 bool Res = selectImpl(I, *CoverageInfo); 285 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); 286 if (Res) 287 return Res; 288 } 289 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg()); 290 I.removeFromParent(); 291 return true; 292 } else if (I.getNumDefs() == 1) { 293 // Make all vregs 32 bits (for SPIR-V IDs). 294 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); 295 } 296 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 297 } 298 299 if (I.getNumOperands() != I.getNumExplicitOperands()) { 300 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); 301 return false; 302 } 303 304 // Common code for getting return reg+type, and removing selected instr 305 // from parent occurs here. Instr-specific selection happens in spvSelect(). 306 bool HasDefs = I.getNumDefs() > 0; 307 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); 308 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; 309 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 310 if (spvSelect(ResVReg, ResType, I)) { 311 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). 312 for (unsigned i = 0; i < I.getNumDefs(); ++i) 313 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32)); 314 I.removeFromParent(); 315 return true; 316 } 317 return false; 318 } 319 320 bool SPIRVInstructionSelector::spvSelect(Register ResVReg, 321 const SPIRVType *ResType, 322 MachineInstr &I) const { 323 const unsigned Opcode = I.getOpcode(); 324 if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT) 325 return selectImpl(I, *CoverageInfo); 326 switch (Opcode) { 327 case TargetOpcode::G_CONSTANT: 328 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), 329 I); 330 case TargetOpcode::G_GLOBAL_VALUE: 331 return selectGlobalValue(ResVReg, I); 332 case TargetOpcode::G_IMPLICIT_DEF: 333 return selectOpUndef(ResVReg, ResType, I); 334 case TargetOpcode::G_FREEZE: 335 return selectFreeze(ResVReg, ResType, I); 336 337 case TargetOpcode::G_INTRINSIC: 338 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 339 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: 340 return selectIntrinsic(ResVReg, ResType, I); 341 case TargetOpcode::G_BITREVERSE: 342 return selectBitreverse(ResVReg, ResType, I); 343 344 case TargetOpcode::G_BUILD_VECTOR: 345 return selectConstVector(ResVReg, ResType, I); 346 case TargetOpcode::G_SPLAT_VECTOR: 347 return selectSplatVector(ResVReg, ResType, I); 348 349 case TargetOpcode::G_SHUFFLE_VECTOR: { 350 MachineBasicBlock &BB = *I.getParent(); 351 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) 352 .addDef(ResVReg) 353 .addUse(GR.getSPIRVTypeID(ResType)) 354 .addUse(I.getOperand(1).getReg()) 355 .addUse(I.getOperand(2).getReg()); 356 for (auto V : I.getOperand(3).getShuffleMask()) 357 MIB.addImm(V); 358 return MIB.constrainAllUses(TII, TRI, RBI); 359 } 360 case TargetOpcode::G_MEMMOVE: 361 case TargetOpcode::G_MEMCPY: 362 case TargetOpcode::G_MEMSET: 363 return selectMemOperation(ResVReg, I); 364 365 case TargetOpcode::G_ICMP: 366 return selectICmp(ResVReg, ResType, I); 367 case TargetOpcode::G_FCMP: 368 return selectFCmp(ResVReg, ResType, I); 369 370 case TargetOpcode::G_FRAME_INDEX: 371 return selectFrameIndex(ResVReg, ResType, I); 372 373 case TargetOpcode::G_LOAD: 374 return selectLoad(ResVReg, ResType, I); 375 case TargetOpcode::G_STORE: 376 return selectStore(I); 377 378 case TargetOpcode::G_BR: 379 return selectBranch(I); 380 case TargetOpcode::G_BRCOND: 381 return selectBranchCond(I); 382 383 case TargetOpcode::G_PHI: 384 return selectPhi(ResVReg, ResType, I); 385 386 case TargetOpcode::G_FPTOSI: 387 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); 388 case TargetOpcode::G_FPTOUI: 389 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); 390 391 case TargetOpcode::G_SITOFP: 392 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); 393 case TargetOpcode::G_UITOFP: 394 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); 395 396 case TargetOpcode::G_CTPOP: 397 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); 398 case TargetOpcode::G_SMIN: 399 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); 400 case TargetOpcode::G_UMIN: 401 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); 402 403 case TargetOpcode::G_SMAX: 404 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); 405 case TargetOpcode::G_UMAX: 406 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); 407 408 case TargetOpcode::G_FMA: 409 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); 410 411 case TargetOpcode::G_FPOW: 412 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); 413 case TargetOpcode::G_FPOWI: 414 return selectExtInst(ResVReg, ResType, I, CL::pown); 415 416 case TargetOpcode::G_FEXP: 417 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); 418 case TargetOpcode::G_FEXP2: 419 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); 420 421 case TargetOpcode::G_FLOG: 422 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); 423 case TargetOpcode::G_FLOG2: 424 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); 425 case TargetOpcode::G_FLOG10: 426 return selectLog10(ResVReg, ResType, I); 427 428 case TargetOpcode::G_FABS: 429 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); 430 case TargetOpcode::G_ABS: 431 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); 432 433 case TargetOpcode::G_FMINNUM: 434 case TargetOpcode::G_FMINIMUM: 435 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); 436 case TargetOpcode::G_FMAXNUM: 437 case TargetOpcode::G_FMAXIMUM: 438 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); 439 440 case TargetOpcode::G_FCOPYSIGN: 441 return selectExtInst(ResVReg, ResType, I, CL::copysign); 442 443 case TargetOpcode::G_FCEIL: 444 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); 445 case TargetOpcode::G_FFLOOR: 446 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); 447 448 case TargetOpcode::G_FCOS: 449 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); 450 case TargetOpcode::G_FSIN: 451 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); 452 453 case TargetOpcode::G_FSQRT: 454 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); 455 456 case TargetOpcode::G_CTTZ: 457 case TargetOpcode::G_CTTZ_ZERO_UNDEF: 458 return selectExtInst(ResVReg, ResType, I, CL::ctz); 459 case TargetOpcode::G_CTLZ: 460 case TargetOpcode::G_CTLZ_ZERO_UNDEF: 461 return selectExtInst(ResVReg, ResType, I, CL::clz); 462 463 case TargetOpcode::G_INTRINSIC_ROUND: 464 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); 465 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: 466 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 467 case TargetOpcode::G_INTRINSIC_TRUNC: 468 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); 469 case TargetOpcode::G_FRINT: 470 case TargetOpcode::G_FNEARBYINT: 471 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); 472 473 case TargetOpcode::G_SMULH: 474 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); 475 case TargetOpcode::G_UMULH: 476 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); 477 478 case TargetOpcode::G_SEXT: 479 return selectExt(ResVReg, ResType, I, true); 480 case TargetOpcode::G_ANYEXT: 481 case TargetOpcode::G_ZEXT: 482 return selectExt(ResVReg, ResType, I, false); 483 case TargetOpcode::G_TRUNC: 484 return selectTrunc(ResVReg, ResType, I); 485 case TargetOpcode::G_FPTRUNC: 486 case TargetOpcode::G_FPEXT: 487 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); 488 489 case TargetOpcode::G_PTRTOINT: 490 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); 491 case TargetOpcode::G_INTTOPTR: 492 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); 493 case TargetOpcode::G_BITCAST: 494 return selectBitcast(ResVReg, ResType, I); 495 case TargetOpcode::G_ADDRSPACE_CAST: 496 return selectAddrSpaceCast(ResVReg, ResType, I); 497 case TargetOpcode::G_PTR_ADD: { 498 // Currently, we get G_PTR_ADD only as a result of translating 499 // global variables, initialized with constant expressions like GV + Const 500 // (see test opencl/basic/progvar_prog_scope_init.ll). 501 // TODO: extend the handler once we have other cases. 502 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 503 Register GV = I.getOperand(1).getReg(); 504 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); 505 (void)II; 506 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || 507 (*II).getOpcode() == TargetOpcode::COPY || 508 (*II).getOpcode() == SPIRV::OpVariable) && 509 isImm(I.getOperand(2), MRI)); 510 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); 511 MachineBasicBlock &BB = *I.getParent(); 512 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 513 .addDef(ResVReg) 514 .addUse(GR.getSPIRVTypeID(ResType)) 515 .addImm(static_cast<uint32_t>( 516 SPIRV::Opcode::InBoundsPtrAccessChain)) 517 .addUse(GV) 518 .addUse(Idx) 519 .addUse(I.getOperand(2).getReg()); 520 return MIB.constrainAllUses(TII, TRI, RBI); 521 } 522 523 case TargetOpcode::G_ATOMICRMW_OR: 524 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); 525 case TargetOpcode::G_ATOMICRMW_ADD: 526 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); 527 case TargetOpcode::G_ATOMICRMW_AND: 528 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); 529 case TargetOpcode::G_ATOMICRMW_MAX: 530 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); 531 case TargetOpcode::G_ATOMICRMW_MIN: 532 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); 533 case TargetOpcode::G_ATOMICRMW_SUB: 534 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); 535 case TargetOpcode::G_ATOMICRMW_XOR: 536 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); 537 case TargetOpcode::G_ATOMICRMW_UMAX: 538 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); 539 case TargetOpcode::G_ATOMICRMW_UMIN: 540 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); 541 case TargetOpcode::G_ATOMICRMW_XCHG: 542 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); 543 case TargetOpcode::G_ATOMIC_CMPXCHG: 544 return selectAtomicCmpXchg(ResVReg, ResType, I); 545 546 case TargetOpcode::G_ATOMICRMW_FADD: 547 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); 548 case TargetOpcode::G_ATOMICRMW_FSUB: 549 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand 550 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, 551 SPIRV::OpFNegate); 552 case TargetOpcode::G_ATOMICRMW_FMIN: 553 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); 554 case TargetOpcode::G_ATOMICRMW_FMAX: 555 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); 556 557 case TargetOpcode::G_FENCE: 558 return selectFence(I); 559 560 case TargetOpcode::G_STACKSAVE: 561 return selectStackSave(ResVReg, ResType, I); 562 case TargetOpcode::G_STACKRESTORE: 563 return selectStackRestore(I); 564 565 case TargetOpcode::G_UNMERGE_VALUES: 566 return selectUnmergeValues(I); 567 568 default: 569 return false; 570 } 571 } 572 573 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 574 const SPIRVType *ResType, 575 MachineInstr &I, 576 CL::OpenCLExtInst CLInst) const { 577 return selectExtInst(ResVReg, ResType, I, 578 {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); 579 } 580 581 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 582 const SPIRVType *ResType, 583 MachineInstr &I, 584 CL::OpenCLExtInst CLInst, 585 GL::GLSLExtInst GLInst) const { 586 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, 587 {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; 588 return selectExtInst(ResVReg, ResType, I, ExtInsts); 589 } 590 591 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, 592 const SPIRVType *ResType, 593 MachineInstr &I, 594 const ExtInstList &Insts) const { 595 596 for (const auto &Ex : Insts) { 597 SPIRV::InstructionSet::InstructionSet Set = Ex.first; 598 uint32_t Opcode = Ex.second; 599 if (STI.canUseExtInstSet(Set)) { 600 MachineBasicBlock &BB = *I.getParent(); 601 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 602 .addDef(ResVReg) 603 .addUse(GR.getSPIRVTypeID(ResType)) 604 .addImm(static_cast<uint32_t>(Set)) 605 .addImm(Opcode); 606 const unsigned NumOps = I.getNumOperands(); 607 for (unsigned i = 1; i < NumOps; ++i) 608 MIB.add(I.getOperand(i)); 609 return MIB.constrainAllUses(TII, TRI, RBI); 610 } 611 } 612 return false; 613 } 614 615 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, 616 const SPIRVType *ResType, 617 MachineInstr &I, 618 Register SrcReg, 619 unsigned Opcode) const { 620 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 621 .addDef(ResVReg) 622 .addUse(GR.getSPIRVTypeID(ResType)) 623 .addUse(SrcReg) 624 .constrainAllUses(TII, TRI, RBI); 625 } 626 627 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, 628 const SPIRVType *ResType, 629 MachineInstr &I, 630 unsigned Opcode) const { 631 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), 632 Opcode); 633 } 634 635 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, 636 const SPIRVType *ResType, 637 MachineInstr &I) const { 638 Register OpReg = I.getOperand(1).getReg(); 639 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; 640 if (!GR.isBitcastCompatible(ResType, OpType)) 641 report_fatal_error("incompatible result and operand types in a bitcast"); 642 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 643 } 644 645 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord, 646 SPIRVMachineModuleInfo *MMI) { 647 if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID) 648 return SPIRV::Scope::Invocation; 649 else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID) 650 return SPIRV::Scope::Device; 651 else if (Ord == MMI->WorkGroupSSID) 652 return SPIRV::Scope::Workgroup; 653 else if (Ord == MMI->AllSVMDevicesSSID) 654 return SPIRV::Scope::CrossDevice; 655 else if (Ord == MMI->SubGroupSSID) 656 return SPIRV::Scope::Subgroup; 657 else 658 // OpenCL approach is: "The functions that do not have memory_scope argument 659 // have the same semantics as the corresponding functions with the 660 // memory_scope argument set to memory_scope_device." See ref.: // 661 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions 662 // In our case if the scope is unknown, assuming that SPIR-V code is to be 663 // consumed in an OpenCL environment, we use the same approach and set the 664 // scope to memory_scope_device. 665 return SPIRV::Scope::Device; 666 } 667 668 static void addMemoryOperands(MachineMemOperand *MemOp, 669 MachineInstrBuilder &MIB) { 670 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 671 if (MemOp->isVolatile()) 672 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 673 if (MemOp->isNonTemporal()) 674 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 675 if (MemOp->getAlign().value()) 676 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); 677 678 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { 679 MIB.addImm(SpvMemOp); 680 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) 681 MIB.addImm(MemOp->getAlign().value()); 682 } 683 } 684 685 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { 686 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); 687 if (Flags & MachineMemOperand::Flags::MOVolatile) 688 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); 689 if (Flags & MachineMemOperand::Flags::MONonTemporal) 690 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); 691 692 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) 693 MIB.addImm(SpvMemOp); 694 } 695 696 bool SPIRVInstructionSelector::selectLoad(Register ResVReg, 697 const SPIRVType *ResType, 698 MachineInstr &I) const { 699 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 700 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 701 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 702 .addDef(ResVReg) 703 .addUse(GR.getSPIRVTypeID(ResType)) 704 .addUse(Ptr); 705 if (!I.getNumMemOperands()) { 706 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 707 I.getOpcode() == 708 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 709 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 710 } else { 711 addMemoryOperands(*I.memoperands_begin(), MIB); 712 } 713 return MIB.constrainAllUses(TII, TRI, RBI); 714 } 715 716 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { 717 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; 718 Register StoreVal = I.getOperand(0 + OpOffset).getReg(); 719 Register Ptr = I.getOperand(1 + OpOffset).getReg(); 720 MachineBasicBlock &BB = *I.getParent(); 721 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) 722 .addUse(Ptr) 723 .addUse(StoreVal); 724 if (!I.getNumMemOperands()) { 725 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || 726 I.getOpcode() == 727 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); 728 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); 729 } else { 730 addMemoryOperands(*I.memoperands_begin(), MIB); 731 } 732 return MIB.constrainAllUses(TII, TRI, RBI); 733 } 734 735 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, 736 const SPIRVType *ResType, 737 MachineInstr &I) const { 738 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 739 report_fatal_error( 740 "llvm.stacksave intrinsic: this instruction requires the following " 741 "SPIR-V extension: SPV_INTEL_variable_length_array", 742 false); 743 MachineBasicBlock &BB = *I.getParent(); 744 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) 745 .addDef(ResVReg) 746 .addUse(GR.getSPIRVTypeID(ResType)) 747 .constrainAllUses(TII, TRI, RBI); 748 } 749 750 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { 751 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) 752 report_fatal_error( 753 "llvm.stackrestore intrinsic: this instruction requires the following " 754 "SPIR-V extension: SPV_INTEL_variable_length_array", 755 false); 756 if (!I.getOperand(0).isReg()) 757 return false; 758 MachineBasicBlock &BB = *I.getParent(); 759 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) 760 .addUse(I.getOperand(0).getReg()) 761 .constrainAllUses(TII, TRI, RBI); 762 } 763 764 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, 765 MachineInstr &I) const { 766 MachineBasicBlock &BB = *I.getParent(); 767 Register SrcReg = I.getOperand(1).getReg(); 768 if (I.getOpcode() == TargetOpcode::G_MEMSET) { 769 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); 770 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); 771 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); 772 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 773 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); 774 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); 775 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( 776 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); 777 // TODO: check if we have such GV, add init, use buildGlobalVariable. 778 Function &CurFunction = GR.CurMF->getFunction(); 779 Type *LLVMArrTy = 780 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); 781 // Module takes ownership of the global var. 782 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, 783 true, GlobalValue::InternalLinkage, 784 Constant::getNullValue(LLVMArrTy)); 785 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 786 GR.add(GV, GR.CurMF, VarReg); 787 788 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); 789 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) 790 .addDef(VarReg) 791 .addUse(GR.getSPIRVTypeID(VarTy)) 792 .addImm(SPIRV::StorageClass::UniformConstant) 793 .addUse(Const) 794 .constrainAllUses(TII, TRI, RBI); 795 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( 796 ValTy, I, TII, SPIRV::StorageClass::UniformConstant); 797 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 798 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); 799 } 800 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) 801 .addUse(I.getOperand(0).getReg()) 802 .addUse(SrcReg) 803 .addUse(I.getOperand(2).getReg()); 804 if (I.getNumMemOperands()) 805 addMemoryOperands(*I.memoperands_begin(), MIB); 806 bool Result = MIB.constrainAllUses(TII, TRI, RBI); 807 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) 808 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) 809 .addUse(MIB->getOperand(0).getReg()); 810 return Result; 811 } 812 813 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, 814 const SPIRVType *ResType, 815 MachineInstr &I, 816 unsigned NewOpcode, 817 unsigned NegateOpcode) const { 818 assert(I.hasOneMemOperand()); 819 const MachineMemOperand *MemOp = *I.memoperands_begin(); 820 uint32_t Scope = 821 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); 822 Register ScopeReg = buildI32Constant(Scope, I); 823 824 Register Ptr = I.getOperand(1).getReg(); 825 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll 826 // auto ScSem = 827 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); 828 AtomicOrdering AO = MemOp->getSuccessOrdering(); 829 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 830 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); 831 832 bool Result = false; 833 Register ValueReg = I.getOperand(2).getReg(); 834 if (NegateOpcode != 0) { 835 // Translation with negative value operand is requested 836 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 837 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); 838 ValueReg = TmpReg; 839 } 840 841 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) 842 .addDef(ResVReg) 843 .addUse(GR.getSPIRVTypeID(ResType)) 844 .addUse(Ptr) 845 .addUse(ScopeReg) 846 .addUse(MemSemReg) 847 .addUse(ValueReg) 848 .constrainAllUses(TII, TRI, RBI); 849 return Result; 850 } 851 852 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { 853 unsigned ArgI = I.getNumOperands() - 1; 854 Register SrcReg = 855 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); 856 SPIRVType *DefType = 857 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; 858 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) 859 report_fatal_error( 860 "cannot select G_UNMERGE_VALUES with a non-vector argument"); 861 862 SPIRVType *ScalarType = 863 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); 864 MachineBasicBlock &BB = *I.getParent(); 865 bool Res = false; 866 for (unsigned i = 0; i < I.getNumDefs(); ++i) { 867 Register ResVReg = I.getOperand(i).getReg(); 868 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); 869 if (!ResType) { 870 // There was no "assign type" actions, let's fix this now 871 ResType = ScalarType; 872 MRI->setRegClass(ResVReg, &SPIRV::IDRegClass); 873 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); 874 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); 875 } 876 auto MIB = 877 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 878 .addDef(ResVReg) 879 .addUse(GR.getSPIRVTypeID(ResType)) 880 .addUse(SrcReg) 881 .addImm(static_cast<int64_t>(i)); 882 Res |= MIB.constrainAllUses(TII, TRI, RBI); 883 } 884 return Res; 885 } 886 887 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { 888 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); 889 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); 890 Register MemSemReg = buildI32Constant(MemSem, I); 891 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); 892 uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI)); 893 Register ScopeReg = buildI32Constant(Scope, I); 894 MachineBasicBlock &BB = *I.getParent(); 895 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) 896 .addUse(ScopeReg) 897 .addUse(MemSemReg) 898 .constrainAllUses(TII, TRI, RBI); 899 } 900 901 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, 902 const SPIRVType *ResType, 903 MachineInstr &I) const { 904 Register ScopeReg; 905 Register MemSemEqReg; 906 Register MemSemNeqReg; 907 Register Ptr = I.getOperand(2).getReg(); 908 if (!isa<GIntrinsic>(I)) { 909 assert(I.hasOneMemOperand()); 910 const MachineMemOperand *MemOp = *I.memoperands_begin(); 911 unsigned Scope = 912 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI)); 913 ScopeReg = buildI32Constant(Scope, I); 914 915 unsigned ScSem = static_cast<uint32_t>( 916 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); 917 AtomicOrdering AO = MemOp->getSuccessOrdering(); 918 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem; 919 MemSemEqReg = buildI32Constant(MemSemEq, I); 920 AtomicOrdering FO = MemOp->getFailureOrdering(); 921 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem; 922 MemSemNeqReg = 923 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); 924 } else { 925 ScopeReg = I.getOperand(5).getReg(); 926 MemSemEqReg = I.getOperand(6).getReg(); 927 MemSemNeqReg = I.getOperand(7).getReg(); 928 } 929 930 Register Cmp = I.getOperand(3).getReg(); 931 Register Val = I.getOperand(4).getReg(); 932 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); 933 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); 934 const DebugLoc &DL = I.getDebugLoc(); 935 bool Result = 936 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) 937 .addDef(ACmpRes) 938 .addUse(GR.getSPIRVTypeID(SpvValTy)) 939 .addUse(Ptr) 940 .addUse(ScopeReg) 941 .addUse(MemSemEqReg) 942 .addUse(MemSemNeqReg) 943 .addUse(Val) 944 .addUse(Cmp) 945 .constrainAllUses(TII, TRI, RBI); 946 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 947 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); 948 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) 949 .addDef(CmpSuccReg) 950 .addUse(GR.getSPIRVTypeID(BoolTy)) 951 .addUse(ACmpRes) 952 .addUse(Cmp) 953 .constrainAllUses(TII, TRI, RBI); 954 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 955 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 956 .addDef(TmpReg) 957 .addUse(GR.getSPIRVTypeID(ResType)) 958 .addUse(ACmpRes) 959 .addUse(GR.getOrCreateUndef(I, ResType, TII)) 960 .addImm(0) 961 .constrainAllUses(TII, TRI, RBI); 962 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) 963 .addDef(ResVReg) 964 .addUse(GR.getSPIRVTypeID(ResType)) 965 .addUse(CmpSuccReg) 966 .addUse(TmpReg) 967 .addImm(1) 968 .constrainAllUses(TII, TRI, RBI); 969 return Result; 970 } 971 972 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { 973 switch (SC) { 974 case SPIRV::StorageClass::Workgroup: 975 case SPIRV::StorageClass::CrossWorkgroup: 976 case SPIRV::StorageClass::Function: 977 return true; 978 default: 979 return false; 980 } 981 } 982 983 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { 984 switch (SC) { 985 case SPIRV::StorageClass::DeviceOnlyINTEL: 986 case SPIRV::StorageClass::HostOnlyINTEL: 987 return true; 988 default: 989 return false; 990 } 991 } 992 993 // In SPIR-V address space casting can only happen to and from the Generic 994 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function 995 // pointers to and from Generic pointers. As such, we can convert e.g. from 996 // Workgroup to Function by going via a Generic pointer as an intermediary. All 997 // other combinations can only be done by a bitcast, and are probably not safe. 998 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, 999 const SPIRVType *ResType, 1000 MachineInstr &I) const { 1001 // If the AddrSpaceCast user is single and in OpConstantComposite or 1002 // OpVariable, we should select OpSpecConstantOp. 1003 auto UIs = MRI->use_instructions(ResVReg); 1004 if (!UIs.empty() && ++UIs.begin() == UIs.end() && 1005 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || 1006 UIs.begin()->getOpcode() == SPIRV::OpVariable || 1007 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { 1008 Register NewReg = I.getOperand(1).getReg(); 1009 MachineBasicBlock &BB = *I.getParent(); 1010 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); 1011 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, 1012 SPIRV::StorageClass::Generic); 1013 bool Result = 1014 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 1015 .addDef(ResVReg) 1016 .addUse(GR.getSPIRVTypeID(ResType)) 1017 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)) 1018 .addUse(NewReg) 1019 .constrainAllUses(TII, TRI, RBI); 1020 return Result; 1021 } 1022 Register SrcPtr = I.getOperand(1).getReg(); 1023 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); 1024 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); 1025 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); 1026 1027 // don't generate a cast between identical storage classes 1028 if (SrcSC == DstSC) 1029 return true; 1030 1031 // Casting from an eligible pointer to Generic. 1032 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) 1033 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); 1034 // Casting from Generic to an eligible pointer. 1035 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) 1036 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); 1037 // Casting between 2 eligible pointers using Generic as an intermediary. 1038 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { 1039 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1040 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( 1041 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic); 1042 MachineBasicBlock &BB = *I.getParent(); 1043 const DebugLoc &DL = I.getDebugLoc(); 1044 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) 1045 .addDef(Tmp) 1046 .addUse(GR.getSPIRVTypeID(GenericPtrTy)) 1047 .addUse(SrcPtr) 1048 .constrainAllUses(TII, TRI, RBI); 1049 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) 1050 .addDef(ResVReg) 1051 .addUse(GR.getSPIRVTypeID(ResType)) 1052 .addUse(Tmp) 1053 .constrainAllUses(TII, TRI, RBI); 1054 } 1055 1056 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may 1057 // be applied 1058 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) 1059 return selectUnOp(ResVReg, ResType, I, 1060 SPIRV::OpPtrCastToCrossWorkgroupINTEL); 1061 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) 1062 return selectUnOp(ResVReg, ResType, I, 1063 SPIRV::OpCrossWorkgroupCastToPtrINTEL); 1064 1065 // TODO Should this case just be disallowed completely? 1066 // We're casting 2 other arbitrary address spaces, so have to bitcast. 1067 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); 1068 } 1069 1070 static unsigned getFCmpOpcode(unsigned PredNum) { 1071 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1072 switch (Pred) { 1073 case CmpInst::FCMP_OEQ: 1074 return SPIRV::OpFOrdEqual; 1075 case CmpInst::FCMP_OGE: 1076 return SPIRV::OpFOrdGreaterThanEqual; 1077 case CmpInst::FCMP_OGT: 1078 return SPIRV::OpFOrdGreaterThan; 1079 case CmpInst::FCMP_OLE: 1080 return SPIRV::OpFOrdLessThanEqual; 1081 case CmpInst::FCMP_OLT: 1082 return SPIRV::OpFOrdLessThan; 1083 case CmpInst::FCMP_ONE: 1084 return SPIRV::OpFOrdNotEqual; 1085 case CmpInst::FCMP_ORD: 1086 return SPIRV::OpOrdered; 1087 case CmpInst::FCMP_UEQ: 1088 return SPIRV::OpFUnordEqual; 1089 case CmpInst::FCMP_UGE: 1090 return SPIRV::OpFUnordGreaterThanEqual; 1091 case CmpInst::FCMP_UGT: 1092 return SPIRV::OpFUnordGreaterThan; 1093 case CmpInst::FCMP_ULE: 1094 return SPIRV::OpFUnordLessThanEqual; 1095 case CmpInst::FCMP_ULT: 1096 return SPIRV::OpFUnordLessThan; 1097 case CmpInst::FCMP_UNE: 1098 return SPIRV::OpFUnordNotEqual; 1099 case CmpInst::FCMP_UNO: 1100 return SPIRV::OpUnordered; 1101 default: 1102 llvm_unreachable("Unknown predicate type for FCmp"); 1103 } 1104 } 1105 1106 static unsigned getICmpOpcode(unsigned PredNum) { 1107 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1108 switch (Pred) { 1109 case CmpInst::ICMP_EQ: 1110 return SPIRV::OpIEqual; 1111 case CmpInst::ICMP_NE: 1112 return SPIRV::OpINotEqual; 1113 case CmpInst::ICMP_SGE: 1114 return SPIRV::OpSGreaterThanEqual; 1115 case CmpInst::ICMP_SGT: 1116 return SPIRV::OpSGreaterThan; 1117 case CmpInst::ICMP_SLE: 1118 return SPIRV::OpSLessThanEqual; 1119 case CmpInst::ICMP_SLT: 1120 return SPIRV::OpSLessThan; 1121 case CmpInst::ICMP_UGE: 1122 return SPIRV::OpUGreaterThanEqual; 1123 case CmpInst::ICMP_UGT: 1124 return SPIRV::OpUGreaterThan; 1125 case CmpInst::ICMP_ULE: 1126 return SPIRV::OpULessThanEqual; 1127 case CmpInst::ICMP_ULT: 1128 return SPIRV::OpULessThan; 1129 default: 1130 llvm_unreachable("Unknown predicate type for ICmp"); 1131 } 1132 } 1133 1134 static unsigned getPtrCmpOpcode(unsigned Pred) { 1135 switch (static_cast<CmpInst::Predicate>(Pred)) { 1136 case CmpInst::ICMP_EQ: 1137 return SPIRV::OpPtrEqual; 1138 case CmpInst::ICMP_NE: 1139 return SPIRV::OpPtrNotEqual; 1140 default: 1141 llvm_unreachable("Unknown predicate type for pointer comparison"); 1142 } 1143 } 1144 1145 // Return the logical operation, or abort if none exists. 1146 static unsigned getBoolCmpOpcode(unsigned PredNum) { 1147 auto Pred = static_cast<CmpInst::Predicate>(PredNum); 1148 switch (Pred) { 1149 case CmpInst::ICMP_EQ: 1150 return SPIRV::OpLogicalEqual; 1151 case CmpInst::ICMP_NE: 1152 return SPIRV::OpLogicalNotEqual; 1153 default: 1154 llvm_unreachable("Unknown predicate type for Bool comparison"); 1155 } 1156 } 1157 1158 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, 1159 const SPIRVType *ResType, 1160 MachineInstr &I) const { 1161 MachineBasicBlock &BB = *I.getParent(); 1162 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) 1163 .addDef(ResVReg) 1164 .addUse(GR.getSPIRVTypeID(ResType)) 1165 .addUse(I.getOperand(1).getReg()) 1166 .constrainAllUses(TII, TRI, RBI); 1167 } 1168 1169 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, 1170 const SPIRVType *ResType, 1171 MachineInstr &I) const { 1172 // There is no way to implement `freeze` correctly without support on SPIR-V 1173 // standard side, but we may at least address a simple (static) case when 1174 // undef/poison value presence is obvious. The main benefit of even 1175 // incomplete `freeze` support is preventing of translation from crashing due 1176 // to lack of support on legalization and instruction selection steps. 1177 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) 1178 return false; 1179 Register OpReg = I.getOperand(1).getReg(); 1180 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { 1181 Register Reg; 1182 switch (Def->getOpcode()) { 1183 case SPIRV::ASSIGN_TYPE: 1184 if (MachineInstr *AssignToDef = 1185 MRI->getVRegDef(Def->getOperand(1).getReg())) { 1186 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) 1187 Reg = Def->getOperand(2).getReg(); 1188 } 1189 break; 1190 case SPIRV::OpUndef: 1191 Reg = Def->getOperand(1).getReg(); 1192 break; 1193 } 1194 unsigned DestOpCode; 1195 if (Reg.isValid()) { 1196 DestOpCode = SPIRV::OpConstantNull; 1197 } else { 1198 DestOpCode = TargetOpcode::COPY; 1199 Reg = OpReg; 1200 } 1201 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) 1202 .addDef(I.getOperand(0).getReg()) 1203 .addUse(Reg) 1204 .constrainAllUses(TII, TRI, RBI); 1205 } 1206 return false; 1207 } 1208 1209 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, 1210 const SPIRVType *ResType, 1211 MachineInstr &I) const { 1212 // TODO: only const case is supported for now. 1213 assert(std::all_of( 1214 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { 1215 if (MO.isDef()) 1216 return true; 1217 if (!MO.isReg()) 1218 return false; 1219 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); 1220 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 1221 ConstTy->getOperand(1).isReg()); 1222 Register ConstReg = ConstTy->getOperand(1).getReg(); 1223 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 1224 assert(Const); 1225 return (Const->getOpcode() == TargetOpcode::G_CONSTANT || 1226 Const->getOpcode() == TargetOpcode::G_FCONSTANT); 1227 })); 1228 1229 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1230 TII.get(SPIRV::OpConstantComposite)) 1231 .addDef(ResVReg) 1232 .addUse(GR.getSPIRVTypeID(ResType)); 1233 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) 1234 MIB.addUse(I.getOperand(i).getReg()); 1235 return MIB.constrainAllUses(TII, TRI, RBI); 1236 } 1237 1238 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, 1239 const SPIRVType *ResType) { 1240 Register OpReg = ResType->getOperand(2).getReg(); 1241 SPIRVType *OpDef = MRI->getVRegDef(OpReg); 1242 if (!OpDef) 1243 return 0; 1244 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1245 OpDef->getOperand(1).isReg()) { 1246 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1247 OpDef = RefDef; 1248 } 1249 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT 1250 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() 1251 : 0; 1252 return N; 1253 } 1254 1255 // Return true if the type represents a constant register 1256 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) { 1257 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && 1258 OpDef->getOperand(1).isReg()) { 1259 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) 1260 OpDef = RefDef; 1261 } 1262 return OpDef->getOpcode() == TargetOpcode::G_CONSTANT || 1263 OpDef->getOpcode() == TargetOpcode::G_FCONSTANT; 1264 } 1265 1266 // Return true if the virtual register represents a constant 1267 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { 1268 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) 1269 return isConstReg(MRI, OpDef); 1270 return false; 1271 } 1272 1273 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, 1274 const SPIRVType *ResType, 1275 MachineInstr &I) const { 1276 unsigned N = 0; 1277 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1278 N = GR.getScalarOrVectorComponentCount(ResType); 1279 else if (ResType->getOpcode() == SPIRV::OpTypeArray) 1280 N = getArrayComponentCount(MRI, ResType); 1281 else 1282 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); 1283 1284 unsigned OpIdx = I.getNumExplicitDefs(); 1285 if (!I.getOperand(OpIdx).isReg()) 1286 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); 1287 1288 // check if we may construct a constant vector 1289 Register OpReg = I.getOperand(OpIdx).getReg(); 1290 bool IsConst = isConstReg(MRI, OpReg); 1291 1292 if (!IsConst && N < 2) 1293 report_fatal_error( 1294 "There must be at least two constituent operands in a vector"); 1295 1296 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), 1297 TII.get(IsConst ? SPIRV::OpConstantComposite 1298 : SPIRV::OpCompositeConstruct)) 1299 .addDef(ResVReg) 1300 .addUse(GR.getSPIRVTypeID(ResType)); 1301 for (unsigned i = 0; i < N; ++i) 1302 MIB.addUse(OpReg); 1303 return MIB.constrainAllUses(TII, TRI, RBI); 1304 } 1305 1306 bool SPIRVInstructionSelector::selectCmp(Register ResVReg, 1307 const SPIRVType *ResType, 1308 unsigned CmpOpc, 1309 MachineInstr &I) const { 1310 Register Cmp0 = I.getOperand(2).getReg(); 1311 Register Cmp1 = I.getOperand(3).getReg(); 1312 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == 1313 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && 1314 "CMP operands should have the same type"); 1315 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) 1316 .addDef(ResVReg) 1317 .addUse(GR.getSPIRVTypeID(ResType)) 1318 .addUse(Cmp0) 1319 .addUse(Cmp1) 1320 .constrainAllUses(TII, TRI, RBI); 1321 } 1322 1323 bool SPIRVInstructionSelector::selectICmp(Register ResVReg, 1324 const SPIRVType *ResType, 1325 MachineInstr &I) const { 1326 auto Pred = I.getOperand(1).getPredicate(); 1327 unsigned CmpOpc; 1328 1329 Register CmpOperand = I.getOperand(2).getReg(); 1330 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) 1331 CmpOpc = getPtrCmpOpcode(Pred); 1332 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) 1333 CmpOpc = getBoolCmpOpcode(Pred); 1334 else 1335 CmpOpc = getICmpOpcode(Pred); 1336 return selectCmp(ResVReg, ResType, CmpOpc, I); 1337 } 1338 1339 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, 1340 const MachineInstr &I, 1341 int OpIdx) const { 1342 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && 1343 "Expected G_FCONSTANT"); 1344 const ConstantFP *FPImm = I.getOperand(1).getFPImm(); 1345 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); 1346 } 1347 1348 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, 1349 const MachineInstr &I, 1350 int OpIdx) const { 1351 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 1352 "Expected G_CONSTANT"); 1353 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); 1354 } 1355 1356 Register 1357 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, 1358 const SPIRVType *ResType) const { 1359 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); 1360 const SPIRVType *SpvI32Ty = 1361 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); 1362 // Find a constant in DT or build a new one. 1363 auto ConstInt = ConstantInt::get(LLVMTy, Val); 1364 Register NewReg = GR.find(ConstInt, GR.CurMF); 1365 if (!NewReg.isValid()) { 1366 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1367 GR.add(ConstInt, GR.CurMF, NewReg); 1368 MachineInstr *MI; 1369 MachineBasicBlock &BB = *I.getParent(); 1370 if (Val == 0) { 1371 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1372 .addDef(NewReg) 1373 .addUse(GR.getSPIRVTypeID(SpvI32Ty)); 1374 } else { 1375 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1376 .addDef(NewReg) 1377 .addUse(GR.getSPIRVTypeID(SpvI32Ty)) 1378 .addImm(APInt(32, Val).getZExtValue()); 1379 } 1380 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); 1381 } 1382 return NewReg; 1383 } 1384 1385 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, 1386 const SPIRVType *ResType, 1387 MachineInstr &I) const { 1388 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); 1389 return selectCmp(ResVReg, ResType, CmpOp, I); 1390 } 1391 1392 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, 1393 MachineInstr &I) const { 1394 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1395 return GR.getOrCreateConsIntVector(0, I, ResType, TII); 1396 return GR.getOrCreateConstInt(0, I, ResType, TII); 1397 } 1398 1399 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, 1400 const SPIRVType *ResType, 1401 MachineInstr &I) const { 1402 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1403 APInt One = 1404 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); 1405 if (ResType->getOpcode() == SPIRV::OpTypeVector) 1406 return GR.getOrCreateConsIntVector(One.getZExtValue(), I, ResType, TII); 1407 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); 1408 } 1409 1410 bool SPIRVInstructionSelector::selectSelect(Register ResVReg, 1411 const SPIRVType *ResType, 1412 MachineInstr &I, 1413 bool IsSigned) const { 1414 // To extend a bool, we need to use OpSelect between constants. 1415 Register ZeroReg = buildZerosVal(ResType, I); 1416 Register OneReg = buildOnesVal(IsSigned, ResType, I); 1417 bool IsScalarBool = 1418 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); 1419 unsigned Opcode = 1420 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; 1421 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1422 .addDef(ResVReg) 1423 .addUse(GR.getSPIRVTypeID(ResType)) 1424 .addUse(I.getOperand(1).getReg()) 1425 .addUse(OneReg) 1426 .addUse(ZeroReg) 1427 .constrainAllUses(TII, TRI, RBI); 1428 } 1429 1430 bool SPIRVInstructionSelector::selectIToF(Register ResVReg, 1431 const SPIRVType *ResType, 1432 MachineInstr &I, bool IsSigned, 1433 unsigned Opcode) const { 1434 Register SrcReg = I.getOperand(1).getReg(); 1435 // We can convert bool value directly to float type without OpConvert*ToF, 1436 // however the translator generates OpSelect+OpConvert*ToF, so we do the same. 1437 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { 1438 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); 1439 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); 1440 if (ResType->getOpcode() == SPIRV::OpTypeVector) { 1441 const unsigned NumElts = ResType->getOperand(2).getImm(); 1442 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); 1443 } 1444 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1445 selectSelect(SrcReg, TmpType, I, false); 1446 } 1447 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); 1448 } 1449 1450 bool SPIRVInstructionSelector::selectExt(Register ResVReg, 1451 const SPIRVType *ResType, 1452 MachineInstr &I, bool IsSigned) const { 1453 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) 1454 return selectSelect(ResVReg, ResType, I, IsSigned); 1455 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1456 return selectUnOp(ResVReg, ResType, I, Opcode); 1457 } 1458 1459 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, 1460 Register ResVReg, 1461 MachineInstr &I, 1462 const SPIRVType *IntTy, 1463 const SPIRVType *BoolTy) const { 1464 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. 1465 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1466 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; 1467 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; 1468 Register Zero = buildZerosVal(IntTy, I); 1469 Register One = buildOnesVal(false, IntTy, I); 1470 MachineBasicBlock &BB = *I.getParent(); 1471 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1472 .addDef(BitIntReg) 1473 .addUse(GR.getSPIRVTypeID(IntTy)) 1474 .addUse(IntReg) 1475 .addUse(One) 1476 .constrainAllUses(TII, TRI, RBI); 1477 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) 1478 .addDef(ResVReg) 1479 .addUse(GR.getSPIRVTypeID(BoolTy)) 1480 .addUse(BitIntReg) 1481 .addUse(Zero) 1482 .constrainAllUses(TII, TRI, RBI); 1483 } 1484 1485 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, 1486 const SPIRVType *ResType, 1487 MachineInstr &I) const { 1488 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { 1489 Register IntReg = I.getOperand(1).getReg(); 1490 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); 1491 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); 1492 } 1493 bool IsSigned = GR.isScalarOrVectorSigned(ResType); 1494 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; 1495 return selectUnOp(ResVReg, ResType, I, Opcode); 1496 } 1497 1498 bool SPIRVInstructionSelector::selectConst(Register ResVReg, 1499 const SPIRVType *ResType, 1500 const APInt &Imm, 1501 MachineInstr &I) const { 1502 unsigned TyOpcode = ResType->getOpcode(); 1503 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); 1504 MachineBasicBlock &BB = *I.getParent(); 1505 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && 1506 Imm.isZero()) 1507 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1508 .addDef(ResVReg) 1509 .addUse(GR.getSPIRVTypeID(ResType)) 1510 .constrainAllUses(TII, TRI, RBI); 1511 if (TyOpcode == SPIRV::OpTypeInt) { 1512 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); 1513 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); 1514 if (Reg == ResVReg) 1515 return true; 1516 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1517 .addDef(ResVReg) 1518 .addUse(Reg) 1519 .constrainAllUses(TII, TRI, RBI); 1520 } 1521 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) 1522 .addDef(ResVReg) 1523 .addUse(GR.getSPIRVTypeID(ResType)); 1524 // <=32-bit integers should be caught by the sdag pattern. 1525 assert(Imm.getBitWidth() > 32); 1526 addNumImm(Imm, MIB); 1527 return MIB.constrainAllUses(TII, TRI, RBI); 1528 } 1529 1530 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, 1531 const SPIRVType *ResType, 1532 MachineInstr &I) const { 1533 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1534 .addDef(ResVReg) 1535 .addUse(GR.getSPIRVTypeID(ResType)) 1536 .constrainAllUses(TII, TRI, RBI); 1537 } 1538 1539 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1540 assert(MO.isReg()); 1541 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1542 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE) 1543 return false; 1544 assert(TypeInst->getOperand(1).isReg()); 1545 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1546 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; 1547 } 1548 1549 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { 1550 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); 1551 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); 1552 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); 1553 return ImmInst->getOperand(1).getCImm()->getZExtValue(); 1554 } 1555 1556 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, 1557 const SPIRVType *ResType, 1558 MachineInstr &I) const { 1559 MachineBasicBlock &BB = *I.getParent(); 1560 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) 1561 .addDef(ResVReg) 1562 .addUse(GR.getSPIRVTypeID(ResType)) 1563 // object to insert 1564 .addUse(I.getOperand(3).getReg()) 1565 // composite to insert into 1566 .addUse(I.getOperand(2).getReg()); 1567 for (unsigned i = 4; i < I.getNumOperands(); i++) 1568 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1569 return MIB.constrainAllUses(TII, TRI, RBI); 1570 } 1571 1572 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, 1573 const SPIRVType *ResType, 1574 MachineInstr &I) const { 1575 MachineBasicBlock &BB = *I.getParent(); 1576 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 1577 .addDef(ResVReg) 1578 .addUse(GR.getSPIRVTypeID(ResType)) 1579 .addUse(I.getOperand(2).getReg()); 1580 for (unsigned i = 3; i < I.getNumOperands(); i++) 1581 MIB.addImm(foldImm(I.getOperand(i), MRI)); 1582 return MIB.constrainAllUses(TII, TRI, RBI); 1583 } 1584 1585 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, 1586 const SPIRVType *ResType, 1587 MachineInstr &I) const { 1588 if (isImm(I.getOperand(4), MRI)) 1589 return selectInsertVal(ResVReg, ResType, I); 1590 MachineBasicBlock &BB = *I.getParent(); 1591 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) 1592 .addDef(ResVReg) 1593 .addUse(GR.getSPIRVTypeID(ResType)) 1594 .addUse(I.getOperand(2).getReg()) 1595 .addUse(I.getOperand(3).getReg()) 1596 .addUse(I.getOperand(4).getReg()) 1597 .constrainAllUses(TII, TRI, RBI); 1598 } 1599 1600 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, 1601 const SPIRVType *ResType, 1602 MachineInstr &I) const { 1603 if (isImm(I.getOperand(3), MRI)) 1604 return selectExtractVal(ResVReg, ResType, I); 1605 MachineBasicBlock &BB = *I.getParent(); 1606 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) 1607 .addDef(ResVReg) 1608 .addUse(GR.getSPIRVTypeID(ResType)) 1609 .addUse(I.getOperand(2).getReg()) 1610 .addUse(I.getOperand(3).getReg()) 1611 .constrainAllUses(TII, TRI, RBI); 1612 } 1613 1614 bool SPIRVInstructionSelector::selectGEP(Register ResVReg, 1615 const SPIRVType *ResType, 1616 MachineInstr &I) const { 1617 const bool IsGEPInBounds = I.getOperand(2).getImm(); 1618 1619 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only 1620 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, 1621 // we have to use Op[InBounds]AccessChain. 1622 const unsigned Opcode = STI.isVulkanEnv() 1623 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain 1624 : SPIRV::OpAccessChain) 1625 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain 1626 : SPIRV::OpPtrAccessChain); 1627 1628 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) 1629 .addDef(ResVReg) 1630 .addUse(GR.getSPIRVTypeID(ResType)) 1631 // Object to get a pointer to. 1632 .addUse(I.getOperand(3).getReg()); 1633 // Adding indices. 1634 const unsigned StartingIndex = 1635 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) 1636 ? 5 1637 : 4; 1638 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) 1639 Res.addUse(I.getOperand(i).getReg()); 1640 return Res.constrainAllUses(TII, TRI, RBI); 1641 } 1642 1643 // Maybe wrap a value into OpSpecConstantOp 1644 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( 1645 MachineInstr &I, SmallVector<Register> &CompositeArgs) const { 1646 bool Result = true; 1647 unsigned Lim = I.getNumExplicitOperands(); 1648 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { 1649 Register OpReg = I.getOperand(i).getReg(); 1650 SPIRVType *OpDefine = MRI->getVRegDef(OpReg); 1651 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); 1652 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || 1653 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) { 1654 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed 1655 // by selectAddrSpaceCast() 1656 CompositeArgs.push_back(OpReg); 1657 continue; 1658 } 1659 MachineFunction *MF = I.getMF(); 1660 Register WrapReg = GR.find(OpDefine, MF); 1661 if (WrapReg.isValid()) { 1662 CompositeArgs.push_back(WrapReg); 1663 continue; 1664 } 1665 // Create a new register for the wrapper 1666 WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 1667 GR.add(OpDefine, MF, WrapReg); 1668 CompositeArgs.push_back(WrapReg); 1669 // Decorate the wrapper register and generate a new instruction 1670 MRI->setType(WrapReg, LLT::pointer(0, 32)); 1671 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); 1672 MachineBasicBlock &BB = *I.getParent(); 1673 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) 1674 .addDef(WrapReg) 1675 .addUse(GR.getSPIRVTypeID(OpType)) 1676 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) 1677 .addUse(OpReg) 1678 .constrainAllUses(TII, TRI, RBI); 1679 if (!Result) 1680 break; 1681 } 1682 return Result; 1683 } 1684 1685 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, 1686 const SPIRVType *ResType, 1687 MachineInstr &I) const { 1688 MachineBasicBlock &BB = *I.getParent(); 1689 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); 1690 switch (IID) { 1691 case Intrinsic::spv_load: 1692 return selectLoad(ResVReg, ResType, I); 1693 case Intrinsic::spv_store: 1694 return selectStore(I); 1695 case Intrinsic::spv_extractv: 1696 return selectExtractVal(ResVReg, ResType, I); 1697 case Intrinsic::spv_insertv: 1698 return selectInsertVal(ResVReg, ResType, I); 1699 case Intrinsic::spv_extractelt: 1700 return selectExtractElt(ResVReg, ResType, I); 1701 case Intrinsic::spv_insertelt: 1702 return selectInsertElt(ResVReg, ResType, I); 1703 case Intrinsic::spv_gep: 1704 return selectGEP(ResVReg, ResType, I); 1705 case Intrinsic::spv_unref_global: 1706 case Intrinsic::spv_init_global: { 1707 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); 1708 MachineInstr *Init = I.getNumExplicitOperands() > 2 1709 ? MRI->getVRegDef(I.getOperand(2).getReg()) 1710 : nullptr; 1711 assert(MI); 1712 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); 1713 } 1714 case Intrinsic::spv_undef: { 1715 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) 1716 .addDef(ResVReg) 1717 .addUse(GR.getSPIRVTypeID(ResType)); 1718 return MIB.constrainAllUses(TII, TRI, RBI); 1719 } 1720 case Intrinsic::spv_const_composite: { 1721 // If no values are attached, the composite is null constant. 1722 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); 1723 // Select a proper instruction. 1724 unsigned Opcode = SPIRV::OpConstantNull; 1725 SmallVector<Register> CompositeArgs; 1726 if (!IsNull) { 1727 Opcode = SPIRV::OpConstantComposite; 1728 if (!wrapIntoSpecConstantOp(I, CompositeArgs)) 1729 return false; 1730 } 1731 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 1732 .addDef(ResVReg) 1733 .addUse(GR.getSPIRVTypeID(ResType)); 1734 // skip type MD node we already used when generated assign.type for this 1735 if (!IsNull) { 1736 for (Register OpReg : CompositeArgs) 1737 MIB.addUse(OpReg); 1738 } 1739 return MIB.constrainAllUses(TII, TRI, RBI); 1740 } 1741 case Intrinsic::spv_assign_name: { 1742 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); 1743 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); 1744 for (unsigned i = I.getNumExplicitDefs() + 2; 1745 i < I.getNumExplicitOperands(); ++i) { 1746 MIB.addImm(I.getOperand(i).getImm()); 1747 } 1748 return MIB.constrainAllUses(TII, TRI, RBI); 1749 } 1750 case Intrinsic::spv_switch: { 1751 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); 1752 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { 1753 if (I.getOperand(i).isReg()) 1754 MIB.addReg(I.getOperand(i).getReg()); 1755 else if (I.getOperand(i).isCImm()) 1756 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); 1757 else if (I.getOperand(i).isMBB()) 1758 MIB.addMBB(I.getOperand(i).getMBB()); 1759 else 1760 llvm_unreachable("Unexpected OpSwitch operand"); 1761 } 1762 return MIB.constrainAllUses(TII, TRI, RBI); 1763 } 1764 case Intrinsic::spv_cmpxchg: 1765 return selectAtomicCmpXchg(ResVReg, ResType, I); 1766 case Intrinsic::spv_unreachable: 1767 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); 1768 break; 1769 case Intrinsic::spv_alloca: 1770 return selectFrameIndex(ResVReg, ResType, I); 1771 case Intrinsic::spv_alloca_array: 1772 return selectAllocaArray(ResVReg, ResType, I); 1773 case Intrinsic::spv_assume: 1774 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1775 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) 1776 .addUse(I.getOperand(1).getReg()); 1777 break; 1778 case Intrinsic::spv_expect: 1779 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) 1780 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) 1781 .addDef(ResVReg) 1782 .addUse(GR.getSPIRVTypeID(ResType)) 1783 .addUse(I.getOperand(2).getReg()) 1784 .addUse(I.getOperand(3).getReg()); 1785 break; 1786 case Intrinsic::spv_thread_id: 1787 return selectSpvThreadId(ResVReg, ResType, I); 1788 case Intrinsic::spv_lifetime_start: 1789 case Intrinsic::spv_lifetime_end: { 1790 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart 1791 : SPIRV::OpLifetimeStop; 1792 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); 1793 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); 1794 unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg); 1795 bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid; 1796 if (Size == -1 || IsNonvoidPtr) 1797 Size = 0; 1798 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); 1799 } break; 1800 default: { 1801 std::string DiagMsg; 1802 raw_string_ostream OS(DiagMsg); 1803 I.print(OS); 1804 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; 1805 report_fatal_error(DiagMsg.c_str(), false); 1806 } 1807 } 1808 return true; 1809 } 1810 1811 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, 1812 const SPIRVType *ResType, 1813 MachineInstr &I) const { 1814 // there was an allocation size parameter to the allocation instruction 1815 // that is not 1 1816 MachineBasicBlock &BB = *I.getParent(); 1817 return BuildMI(BB, I, I.getDebugLoc(), 1818 TII.get(SPIRV::OpVariableLengthArrayINTEL)) 1819 .addDef(ResVReg) 1820 .addUse(GR.getSPIRVTypeID(ResType)) 1821 .addUse(I.getOperand(2).getReg()) 1822 .constrainAllUses(TII, TRI, RBI); 1823 } 1824 1825 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, 1826 const SPIRVType *ResType, 1827 MachineInstr &I) const { 1828 // Change order of instructions if needed: all OpVariable instructions in a 1829 // function must be the first instructions in the first block 1830 MachineFunction *MF = I.getParent()->getParent(); 1831 MachineBasicBlock *MBB = &MF->front(); 1832 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); 1833 bool IsHeader = false; 1834 unsigned Opcode; 1835 for (; It != E && It != I; ++It) { 1836 Opcode = It->getOpcode(); 1837 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { 1838 IsHeader = true; 1839 } else if (IsHeader && 1840 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { 1841 ++It; 1842 break; 1843 } 1844 } 1845 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) 1846 .addDef(ResVReg) 1847 .addUse(GR.getSPIRVTypeID(ResType)) 1848 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)) 1849 .constrainAllUses(TII, TRI, RBI); 1850 } 1851 1852 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { 1853 // InstructionSelector walks backwards through the instructions. We can use 1854 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR 1855 // first, so can generate an OpBranchConditional here. If there is no 1856 // G_BRCOND, we just use OpBranch for a regular unconditional branch. 1857 const MachineInstr *PrevI = I.getPrevNode(); 1858 MachineBasicBlock &MBB = *I.getParent(); 1859 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { 1860 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1861 .addUse(PrevI->getOperand(0).getReg()) 1862 .addMBB(PrevI->getOperand(1).getMBB()) 1863 .addMBB(I.getOperand(0).getMBB()) 1864 .constrainAllUses(TII, TRI, RBI); 1865 } 1866 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) 1867 .addMBB(I.getOperand(0).getMBB()) 1868 .constrainAllUses(TII, TRI, RBI); 1869 } 1870 1871 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { 1872 // InstructionSelector walks backwards through the instructions. For an 1873 // explicit conditional branch with no fallthrough, we use both a G_BR and a 1874 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and 1875 // generate the OpBranchConditional in selectBranch above. 1876 // 1877 // If an OpBranchConditional has been generated, we simply return, as the work 1878 // is alread done. If there is no OpBranchConditional, LLVM must be relying on 1879 // implicit fallthrough to the next basic block, so we need to create an 1880 // OpBranchConditional with an explicit "false" argument pointing to the next 1881 // basic block that LLVM would fall through to. 1882 const MachineInstr *NextI = I.getNextNode(); 1883 // Check if this has already been successfully selected. 1884 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) 1885 return true; 1886 // Must be relying on implicit block fallthrough, so generate an 1887 // OpBranchConditional with the "next" basic block as the "false" target. 1888 MachineBasicBlock &MBB = *I.getParent(); 1889 unsigned NextMBBNum = MBB.getNextNode()->getNumber(); 1890 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); 1891 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) 1892 .addUse(I.getOperand(0).getReg()) 1893 .addMBB(I.getOperand(1).getMBB()) 1894 .addMBB(NextMBB) 1895 .constrainAllUses(TII, TRI, RBI); 1896 } 1897 1898 bool SPIRVInstructionSelector::selectPhi(Register ResVReg, 1899 const SPIRVType *ResType, 1900 MachineInstr &I) const { 1901 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) 1902 .addDef(ResVReg) 1903 .addUse(GR.getSPIRVTypeID(ResType)); 1904 const unsigned NumOps = I.getNumOperands(); 1905 for (unsigned i = 1; i < NumOps; i += 2) { 1906 MIB.addUse(I.getOperand(i + 0).getReg()); 1907 MIB.addMBB(I.getOperand(i + 1).getMBB()); 1908 } 1909 return MIB.constrainAllUses(TII, TRI, RBI); 1910 } 1911 1912 bool SPIRVInstructionSelector::selectGlobalValue( 1913 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { 1914 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. 1915 MachineIRBuilder MIRBuilder(I); 1916 const GlobalValue *GV = I.getOperand(1).getGlobal(); 1917 Type *GVType = GR.getDeducedGlobalValueType(GV); 1918 SPIRVType *PointerBaseType; 1919 if (GVType->isArrayTy()) { 1920 SPIRVType *ArrayElementType = 1921 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, 1922 SPIRV::AccessQualifier::ReadWrite, false); 1923 PointerBaseType = GR.getOrCreateSPIRVArrayType( 1924 ArrayElementType, GVType->getArrayNumElements(), I, TII); 1925 } else { 1926 PointerBaseType = GR.getOrCreateSPIRVType( 1927 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); 1928 } 1929 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( 1930 PointerBaseType, I, TII, 1931 addressSpaceToStorageClass(GV->getAddressSpace(), STI)); 1932 1933 std::string GlobalIdent; 1934 if (!GV->hasName()) { 1935 unsigned &ID = UnnamedGlobalIDs[GV]; 1936 if (ID == 0) 1937 ID = UnnamedGlobalIDs.size(); 1938 GlobalIdent = "__unnamed_" + Twine(ID).str(); 1939 } else { 1940 GlobalIdent = GV->getGlobalIdentifier(); 1941 } 1942 1943 // Behaviour of functions as operands depends on availability of the 1944 // corresponding extension (SPV_INTEL_function_pointers): 1945 // - If there is an extension to operate with functions as operands: 1946 // We create a proper constant operand and evaluate a correct type for a 1947 // function pointer. 1948 // - Without the required extension: 1949 // We have functions as operands in tests with blocks of instruction e.g. in 1950 // transcoding/global_block.ll. These operands are not used and should be 1951 // substituted by zero constants. Their type is expected to be always 1952 // OpTypePointer Function %uchar. 1953 if (isa<Function>(GV)) { 1954 const Constant *ConstVal = GV; 1955 MachineBasicBlock &BB = *I.getParent(); 1956 Register NewReg = GR.find(ConstVal, GR.CurMF); 1957 if (!NewReg.isValid()) { 1958 Register NewReg = ResVReg; 1959 GR.add(ConstVal, GR.CurMF, NewReg); 1960 const Function *GVFun = 1961 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) 1962 ? dyn_cast<Function>(GV) 1963 : nullptr; 1964 if (GVFun) { 1965 // References to a function via function pointers generate virtual 1966 // registers without a definition. We will resolve it later, during 1967 // module analysis stage. 1968 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 1969 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); 1970 MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); 1971 MachineInstrBuilder MB = 1972 BuildMI(BB, I, I.getDebugLoc(), 1973 TII.get(SPIRV::OpConstantFunctionPointerINTEL)) 1974 .addDef(NewReg) 1975 .addUse(GR.getSPIRVTypeID(ResType)) 1976 .addUse(FuncVReg); 1977 // mapping the function pointer to the used Function 1978 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); 1979 return MB.constrainAllUses(TII, TRI, RBI); 1980 } 1981 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) 1982 .addDef(NewReg) 1983 .addUse(GR.getSPIRVTypeID(ResType)) 1984 .constrainAllUses(TII, TRI, RBI); 1985 } 1986 assert(NewReg != ResVReg); 1987 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) 1988 .addDef(ResVReg) 1989 .addUse(NewReg) 1990 .constrainAllUses(TII, TRI, RBI); 1991 } 1992 auto GlobalVar = cast<GlobalVariable>(GV); 1993 assert(GlobalVar->getName() != "llvm.global.annotations"); 1994 1995 bool HasInit = GlobalVar->hasInitializer() && 1996 !isa<UndefValue>(GlobalVar->getInitializer()); 1997 // Skip empty declaration for GVs with initilaizers till we get the decl with 1998 // passed initializer. 1999 if (HasInit && !Init) 2000 return true; 2001 2002 unsigned AddrSpace = GV->getAddressSpace(); 2003 SPIRV::StorageClass::StorageClass Storage = 2004 addressSpaceToStorageClass(AddrSpace, STI); 2005 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && 2006 Storage != SPIRV::StorageClass::Function; 2007 SPIRV::LinkageType::LinkageType LnkType = 2008 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) 2009 ? SPIRV::LinkageType::Import 2010 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && 2011 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) 2012 ? SPIRV::LinkageType::LinkOnceODR 2013 : SPIRV::LinkageType::Export); 2014 2015 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, 2016 Storage, Init, GlobalVar->isConstant(), 2017 HasLnkTy, LnkType, MIRBuilder, true); 2018 return Reg.isValid(); 2019 } 2020 2021 bool SPIRVInstructionSelector::selectLog10(Register ResVReg, 2022 const SPIRVType *ResType, 2023 MachineInstr &I) const { 2024 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { 2025 return selectExtInst(ResVReg, ResType, I, CL::log10); 2026 } 2027 2028 // There is no log10 instruction in the GLSL Extended Instruction set, so it 2029 // is implemented as: 2030 // log10(x) = log2(x) * (1 / log2(10)) 2031 // = log2(x) * 0.30103 2032 2033 MachineIRBuilder MIRBuilder(I); 2034 MachineBasicBlock &BB = *I.getParent(); 2035 2036 // Build log2(x). 2037 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2038 bool Result = 2039 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) 2040 .addDef(VarReg) 2041 .addUse(GR.getSPIRVTypeID(ResType)) 2042 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) 2043 .addImm(GL::Log2) 2044 .add(I.getOperand(1)) 2045 .constrainAllUses(TII, TRI, RBI); 2046 2047 // Build 0.30103. 2048 assert(ResType->getOpcode() == SPIRV::OpTypeVector || 2049 ResType->getOpcode() == SPIRV::OpTypeFloat); 2050 // TODO: Add matrix implementation once supported by the HLSL frontend. 2051 const SPIRVType *SpirvScalarType = 2052 ResType->getOpcode() == SPIRV::OpTypeVector 2053 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) 2054 : ResType; 2055 Register ScaleReg = 2056 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); 2057 2058 // Multiply log2(x) by 0.30103 to get log10(x) result. 2059 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector 2060 ? SPIRV::OpVectorTimesScalar 2061 : SPIRV::OpFMulS; 2062 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) 2063 .addDef(ResVReg) 2064 .addUse(GR.getSPIRVTypeID(ResType)) 2065 .addUse(VarReg) 2066 .addUse(ScaleReg) 2067 .constrainAllUses(TII, TRI, RBI); 2068 2069 return Result; 2070 } 2071 2072 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, 2073 const SPIRVType *ResType, 2074 MachineInstr &I) const { 2075 // DX intrinsic: @llvm.dx.thread.id(i32) 2076 // ID Name Description 2077 // 93 ThreadId reads the thread ID 2078 2079 MachineIRBuilder MIRBuilder(I); 2080 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); 2081 const SPIRVType *Vec3Ty = 2082 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); 2083 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( 2084 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); 2085 2086 // Create new register for GlobalInvocationID builtin variable. 2087 Register NewRegister = 2088 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); 2089 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32)); 2090 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); 2091 2092 // Build GlobalInvocationID global variable with the necessary decorations. 2093 Register Variable = GR.buildGlobalVariable( 2094 NewRegister, PtrType, 2095 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, 2096 SPIRV::StorageClass::Input, nullptr, true, true, 2097 SPIRV::LinkageType::Import, MIRBuilder, false); 2098 2099 // Create new register for loading value. 2100 MachineRegisterInfo *MRI = MIRBuilder.getMRI(); 2101 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass); 2102 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32)); 2103 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); 2104 2105 // Load v3uint value from the global variable. 2106 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) 2107 .addDef(LoadedRegister) 2108 .addUse(GR.getSPIRVTypeID(Vec3Ty)) 2109 .addUse(Variable); 2110 2111 // Get Thread ID index. Expecting operand is a constant immediate value, 2112 // wrapped in a type assignment. 2113 assert(I.getOperand(2).isReg()); 2114 Register ThreadIdReg = I.getOperand(2).getReg(); 2115 SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg); 2116 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && 2117 ConstTy->getOperand(1).isReg()); 2118 Register ConstReg = ConstTy->getOperand(1).getReg(); 2119 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); 2120 assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); 2121 const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue(); 2122 const uint32_t ThreadId = Val.getZExtValue(); 2123 2124 // Extract the thread ID from the loaded vector value. 2125 MachineBasicBlock &BB = *I.getParent(); 2126 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) 2127 .addDef(ResVReg) 2128 .addUse(GR.getSPIRVTypeID(ResType)) 2129 .addUse(LoadedRegister) 2130 .addImm(ThreadId); 2131 return MIB.constrainAllUses(TII, TRI, RBI); 2132 } 2133 2134 namespace llvm { 2135 InstructionSelector * 2136 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, 2137 const SPIRVSubtarget &Subtarget, 2138 const RegisterBankInfo &RBI) { 2139 return new SPIRVInstructionSelector(TM, Subtarget, RBI); 2140 } 2141 } // namespace llvm 2142