xref: /freebsd-src/contrib/llvm-project/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583) 
10b57cec5SDimitry Andric //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file defines an instruction selector for the NVPTX target.
100b57cec5SDimitry Andric //
110b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
120b57cec5SDimitry Andric 
130b57cec5SDimitry Andric #include "NVPTXISelDAGToDAG.h"
140b57cec5SDimitry Andric #include "MCTargetDesc/NVPTXBaseInfo.h"
15480093f4SDimitry Andric #include "NVPTXUtilities.h"
160b57cec5SDimitry Andric #include "llvm/Analysis/ValueTracking.h"
175f757f3fSDimitry Andric #include "llvm/CodeGen/ISDOpcodes.h"
180b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
190b57cec5SDimitry Andric #include "llvm/IR/Instructions.h"
20480093f4SDimitry Andric #include "llvm/IR/IntrinsicsNVPTX.h"
210b57cec5SDimitry Andric #include "llvm/Support/AtomicOrdering.h"
220b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
230b57cec5SDimitry Andric #include "llvm/Support/Debug.h"
240b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
250b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
260b57cec5SDimitry Andric #include "llvm/Target/TargetIntrinsicInfo.h"
270b57cec5SDimitry Andric 
280b57cec5SDimitry Andric using namespace llvm;
290b57cec5SDimitry Andric 
300b57cec5SDimitry Andric #define DEBUG_TYPE "nvptx-isel"
31bdd1243dSDimitry Andric #define PASS_NAME "NVPTX DAG->DAG Pattern Instruction Selection"
320b57cec5SDimitry Andric 
33*0fca6ea1SDimitry Andric static cl::opt<bool>
34*0fca6ea1SDimitry Andric     EnableRsqrtOpt("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden,
35*0fca6ea1SDimitry Andric                    cl::desc("Enable reciprocal sqrt optimization"));
36*0fca6ea1SDimitry Andric 
370b57cec5SDimitry Andric /// createNVPTXISelDag - This pass converts a legalized DAG into a
380b57cec5SDimitry Andric /// NVPTX-specific DAG, ready for instruction scheduling.
390b57cec5SDimitry Andric FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
405f757f3fSDimitry Andric                                        llvm::CodeGenOptLevel OptLevel) {
41*0fca6ea1SDimitry Andric   return new NVPTXDAGToDAGISelLegacy(TM, OptLevel);
420b57cec5SDimitry Andric }
430b57cec5SDimitry Andric 
44*0fca6ea1SDimitry Andric NVPTXDAGToDAGISelLegacy::NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm,
45*0fca6ea1SDimitry Andric                                                  CodeGenOptLevel OptLevel)
46*0fca6ea1SDimitry Andric     : SelectionDAGISelLegacy(
47*0fca6ea1SDimitry Andric           ID, std::make_unique<NVPTXDAGToDAGISel>(tm, OptLevel)) {}
48bdd1243dSDimitry Andric 
49*0fca6ea1SDimitry Andric char NVPTXDAGToDAGISelLegacy::ID = 0;
50*0fca6ea1SDimitry Andric 
51*0fca6ea1SDimitry Andric INITIALIZE_PASS(NVPTXDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
52bdd1243dSDimitry Andric 
530b57cec5SDimitry Andric NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
545f757f3fSDimitry Andric                                      CodeGenOptLevel OptLevel)
55*0fca6ea1SDimitry Andric     : SelectionDAGISel(tm, OptLevel), TM(tm) {
565f757f3fSDimitry Andric   doMulWide = (OptLevel > CodeGenOptLevel::None);
570b57cec5SDimitry Andric }
580b57cec5SDimitry Andric 
590b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
6081ad6265SDimitry Andric   Subtarget = &MF.getSubtarget<NVPTXSubtarget>();
610b57cec5SDimitry Andric   return SelectionDAGISel::runOnMachineFunction(MF);
620b57cec5SDimitry Andric }
630b57cec5SDimitry Andric 
640b57cec5SDimitry Andric int NVPTXDAGToDAGISel::getDivF32Level() const {
650b57cec5SDimitry Andric   return Subtarget->getTargetLowering()->getDivF32Level();
660b57cec5SDimitry Andric }
670b57cec5SDimitry Andric 
680b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
690b57cec5SDimitry Andric   return Subtarget->getTargetLowering()->usePrecSqrtF32();
700b57cec5SDimitry Andric }
710b57cec5SDimitry Andric 
720b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::useF32FTZ() const {
730b57cec5SDimitry Andric   return Subtarget->getTargetLowering()->useF32FTZ(*MF);
740b57cec5SDimitry Andric }
750b57cec5SDimitry Andric 
760b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::allowFMA() const {
770b57cec5SDimitry Andric   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
780b57cec5SDimitry Andric   return TL->allowFMA(*MF, OptLevel);
790b57cec5SDimitry Andric }
800b57cec5SDimitry Andric 
810b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::allowUnsafeFPMath() const {
820b57cec5SDimitry Andric   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
830b57cec5SDimitry Andric   return TL->allowUnsafeFPMath(*MF);
840b57cec5SDimitry Andric }
850b57cec5SDimitry Andric 
86*0fca6ea1SDimitry Andric bool NVPTXDAGToDAGISel::doRsqrtOpt() const { return EnableRsqrtOpt; }
870b57cec5SDimitry Andric 
880b57cec5SDimitry Andric /// Select - Select instructions not customized! Used for
890b57cec5SDimitry Andric /// expanded, promoted and normal instructions.
900b57cec5SDimitry Andric void NVPTXDAGToDAGISel::Select(SDNode *N) {
910b57cec5SDimitry Andric 
920b57cec5SDimitry Andric   if (N->isMachineOpcode()) {
930b57cec5SDimitry Andric     N->setNodeId(-1);
940b57cec5SDimitry Andric     return; // Already selected.
950b57cec5SDimitry Andric   }
960b57cec5SDimitry Andric 
970b57cec5SDimitry Andric   switch (N->getOpcode()) {
980b57cec5SDimitry Andric   case ISD::LOAD:
990b57cec5SDimitry Andric   case ISD::ATOMIC_LOAD:
1000b57cec5SDimitry Andric     if (tryLoad(N))
1010b57cec5SDimitry Andric       return;
1020b57cec5SDimitry Andric     break;
1030b57cec5SDimitry Andric   case ISD::STORE:
1040b57cec5SDimitry Andric   case ISD::ATOMIC_STORE:
1050b57cec5SDimitry Andric     if (tryStore(N))
1060b57cec5SDimitry Andric       return;
1070b57cec5SDimitry Andric     break;
1080b57cec5SDimitry Andric   case ISD::EXTRACT_VECTOR_ELT:
1090b57cec5SDimitry Andric     if (tryEXTRACT_VECTOR_ELEMENT(N))
1100b57cec5SDimitry Andric       return;
1110b57cec5SDimitry Andric     break;
1120b57cec5SDimitry Andric   case NVPTXISD::SETP_F16X2:
1130b57cec5SDimitry Andric     SelectSETP_F16X2(N);
1140b57cec5SDimitry Andric     return;
1155f757f3fSDimitry Andric   case NVPTXISD::SETP_BF16X2:
1165f757f3fSDimitry Andric     SelectSETP_BF16X2(N);
1175f757f3fSDimitry Andric     return;
1180b57cec5SDimitry Andric   case NVPTXISD::LoadV2:
1190b57cec5SDimitry Andric   case NVPTXISD::LoadV4:
1200b57cec5SDimitry Andric     if (tryLoadVector(N))
1210b57cec5SDimitry Andric       return;
1220b57cec5SDimitry Andric     break;
1230b57cec5SDimitry Andric   case NVPTXISD::LDGV2:
1240b57cec5SDimitry Andric   case NVPTXISD::LDGV4:
1250b57cec5SDimitry Andric   case NVPTXISD::LDUV2:
1260b57cec5SDimitry Andric   case NVPTXISD::LDUV4:
1270b57cec5SDimitry Andric     if (tryLDGLDU(N))
1280b57cec5SDimitry Andric       return;
1290b57cec5SDimitry Andric     break;
1300b57cec5SDimitry Andric   case NVPTXISD::StoreV2:
1310b57cec5SDimitry Andric   case NVPTXISD::StoreV4:
1320b57cec5SDimitry Andric     if (tryStoreVector(N))
1330b57cec5SDimitry Andric       return;
1340b57cec5SDimitry Andric     break;
1350b57cec5SDimitry Andric   case NVPTXISD::LoadParam:
1360b57cec5SDimitry Andric   case NVPTXISD::LoadParamV2:
1370b57cec5SDimitry Andric   case NVPTXISD::LoadParamV4:
1380b57cec5SDimitry Andric     if (tryLoadParam(N))
1390b57cec5SDimitry Andric       return;
1400b57cec5SDimitry Andric     break;
1410b57cec5SDimitry Andric   case NVPTXISD::StoreRetval:
1420b57cec5SDimitry Andric   case NVPTXISD::StoreRetvalV2:
1430b57cec5SDimitry Andric   case NVPTXISD::StoreRetvalV4:
1440b57cec5SDimitry Andric     if (tryStoreRetval(N))
1450b57cec5SDimitry Andric       return;
1460b57cec5SDimitry Andric     break;
1470b57cec5SDimitry Andric   case NVPTXISD::StoreParam:
1480b57cec5SDimitry Andric   case NVPTXISD::StoreParamV2:
1490b57cec5SDimitry Andric   case NVPTXISD::StoreParamV4:
1500b57cec5SDimitry Andric   case NVPTXISD::StoreParamS32:
1510b57cec5SDimitry Andric   case NVPTXISD::StoreParamU32:
1520b57cec5SDimitry Andric     if (tryStoreParam(N))
1530b57cec5SDimitry Andric       return;
1540b57cec5SDimitry Andric     break;
1550b57cec5SDimitry Andric   case ISD::INTRINSIC_WO_CHAIN:
1560b57cec5SDimitry Andric     if (tryIntrinsicNoChain(N))
1570b57cec5SDimitry Andric       return;
1580b57cec5SDimitry Andric     break;
1590b57cec5SDimitry Andric   case ISD::INTRINSIC_W_CHAIN:
1600b57cec5SDimitry Andric     if (tryIntrinsicChain(N))
1610b57cec5SDimitry Andric       return;
1620b57cec5SDimitry Andric     break;
1630b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatS32:
1640b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatFloat:
1650b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatFloatLevel:
1660b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatFloatGrad:
1670b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32S32:
1680b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32Float:
1690b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32FloatLevel:
1700b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32FloatGrad:
1710b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32S32:
1720b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32Float:
1730b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32FloatLevel:
1740b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32FloatGrad:
1750b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatS32:
1760b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatFloat:
1770b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatFloatLevel:
1780b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatFloatGrad:
1790b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32S32:
1800b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32Float:
1810b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32FloatLevel:
1820b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32FloatGrad:
1830b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32S32:
1840b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32Float:
1850b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32FloatLevel:
1860b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32FloatGrad:
1870b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatS32:
1880b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatFloat:
1890b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatFloatLevel:
1900b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatFloatGrad:
1910b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32S32:
1920b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32Float:
1930b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32FloatLevel:
1940b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32FloatGrad:
1950b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32S32:
1960b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32Float:
1970b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32FloatLevel:
1980b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32FloatGrad:
1990b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatS32:
2000b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatFloat:
2010b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatFloatLevel:
2020b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatFloatGrad:
2030b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32S32:
2040b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32Float:
2050b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32FloatLevel:
2060b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32FloatGrad:
2070b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32S32:
2080b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32Float:
2090b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32FloatLevel:
2100b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32FloatGrad:
2110b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatS32:
2120b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatFloat:
2130b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatFloatLevel:
2140b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatFloatGrad:
2150b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32S32:
2160b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32Float:
2170b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32FloatLevel:
2180b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32FloatGrad:
2190b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32S32:
2200b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32Float:
2210b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32FloatLevel:
2220b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32FloatGrad:
2230b57cec5SDimitry Andric   case NVPTXISD::TexCubeFloatFloat:
2240b57cec5SDimitry Andric   case NVPTXISD::TexCubeFloatFloatLevel:
2250b57cec5SDimitry Andric   case NVPTXISD::TexCubeS32Float:
2260b57cec5SDimitry Andric   case NVPTXISD::TexCubeS32FloatLevel:
2270b57cec5SDimitry Andric   case NVPTXISD::TexCubeU32Float:
2280b57cec5SDimitry Andric   case NVPTXISD::TexCubeU32FloatLevel:
2290b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayFloatFloat:
2300b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayFloatFloatLevel:
2310b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayS32Float:
2320b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayS32FloatLevel:
2330b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayU32Float:
2340b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayU32FloatLevel:
2350b57cec5SDimitry Andric   case NVPTXISD::Tld4R2DFloatFloat:
2360b57cec5SDimitry Andric   case NVPTXISD::Tld4G2DFloatFloat:
2370b57cec5SDimitry Andric   case NVPTXISD::Tld4B2DFloatFloat:
2380b57cec5SDimitry Andric   case NVPTXISD::Tld4A2DFloatFloat:
2390b57cec5SDimitry Andric   case NVPTXISD::Tld4R2DS64Float:
2400b57cec5SDimitry Andric   case NVPTXISD::Tld4G2DS64Float:
2410b57cec5SDimitry Andric   case NVPTXISD::Tld4B2DS64Float:
2420b57cec5SDimitry Andric   case NVPTXISD::Tld4A2DS64Float:
2430b57cec5SDimitry Andric   case NVPTXISD::Tld4R2DU64Float:
2440b57cec5SDimitry Andric   case NVPTXISD::Tld4G2DU64Float:
2450b57cec5SDimitry Andric   case NVPTXISD::Tld4B2DU64Float:
2460b57cec5SDimitry Andric   case NVPTXISD::Tld4A2DU64Float:
2470b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatS32:
2480b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatFloat:
2490b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatFloatLevel:
2500b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatFloatGrad:
2510b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32S32:
2520b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32Float:
2530b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32FloatLevel:
2540b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32FloatGrad:
2550b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32S32:
2560b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32Float:
2570b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32FloatLevel:
2580b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32FloatGrad:
2590b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatS32:
2600b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatFloat:
2610b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
2620b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
2630b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32S32:
2640b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32Float:
2650b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
2660b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
2670b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32S32:
2680b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32Float:
2690b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
2700b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
2710b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatS32:
2720b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatFloat:
2730b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatFloatLevel:
2740b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatFloatGrad:
2750b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32S32:
2760b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32Float:
2770b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32FloatLevel:
2780b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32FloatGrad:
2790b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32S32:
2800b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32Float:
2810b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32FloatLevel:
2820b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32FloatGrad:
2830b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatS32:
2840b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatFloat:
2850b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
2860b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
2870b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32S32:
2880b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32Float:
2890b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
2900b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
2910b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32S32:
2920b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32Float:
2930b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
2940b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
2950b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatS32:
2960b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatFloat:
2970b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatFloatLevel:
2980b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatFloatGrad:
2990b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32S32:
3000b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32Float:
3010b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32FloatLevel:
3020b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32FloatGrad:
3030b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32S32:
3040b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32Float:
3050b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32FloatLevel:
3060b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32FloatGrad:
3070b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeFloatFloat:
3080b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3090b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeS32Float:
3100b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3110b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeU32Float:
3120b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3130b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3140b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3150b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayS32Float:
3160b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3170b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayU32Float:
3180b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3197a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeFloatFloatGrad:
3207a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeS32FloatGrad:
3217a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeU32FloatGrad:
3227a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayFloatFloatGrad:
3237a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayS32FloatGrad:
3247a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayU32FloatGrad:
3250b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3260b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3270b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3280b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3290b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedR2DS64Float:
3300b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedG2DS64Float:
3310b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedB2DS64Float:
3320b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedA2DS64Float:
3330b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedR2DU64Float:
3340b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedG2DU64Float:
3350b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedB2DU64Float:
3360b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedA2DU64Float:
3370b57cec5SDimitry Andric     if (tryTextureIntrinsic(N))
3380b57cec5SDimitry Andric       return;
3390b57cec5SDimitry Andric     break;
3400b57cec5SDimitry Andric   case NVPTXISD::Suld1DI8Clamp:
3410b57cec5SDimitry Andric   case NVPTXISD::Suld1DI16Clamp:
3420b57cec5SDimitry Andric   case NVPTXISD::Suld1DI32Clamp:
3430b57cec5SDimitry Andric   case NVPTXISD::Suld1DI64Clamp:
3440b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I8Clamp:
3450b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I16Clamp:
3460b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I32Clamp:
3470b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I64Clamp:
3480b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I8Clamp:
3490b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I16Clamp:
3500b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I32Clamp:
3510b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI8Clamp:
3520b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI16Clamp:
3530b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI32Clamp:
3540b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI64Clamp:
3550b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I8Clamp:
3560b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I16Clamp:
3570b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I32Clamp:
3580b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I64Clamp:
3590b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I8Clamp:
3600b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I16Clamp:
3610b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I32Clamp:
3620b57cec5SDimitry Andric   case NVPTXISD::Suld2DI8Clamp:
3630b57cec5SDimitry Andric   case NVPTXISD::Suld2DI16Clamp:
3640b57cec5SDimitry Andric   case NVPTXISD::Suld2DI32Clamp:
3650b57cec5SDimitry Andric   case NVPTXISD::Suld2DI64Clamp:
3660b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I8Clamp:
3670b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I16Clamp:
3680b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I32Clamp:
3690b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I64Clamp:
3700b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I8Clamp:
3710b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I16Clamp:
3720b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I32Clamp:
3730b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI8Clamp:
3740b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI16Clamp:
3750b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI32Clamp:
3760b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI64Clamp:
3770b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I8Clamp:
3780b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I16Clamp:
3790b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I32Clamp:
3800b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I64Clamp:
3810b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I8Clamp:
3820b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I16Clamp:
3830b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I32Clamp:
3840b57cec5SDimitry Andric   case NVPTXISD::Suld3DI8Clamp:
3850b57cec5SDimitry Andric   case NVPTXISD::Suld3DI16Clamp:
3860b57cec5SDimitry Andric   case NVPTXISD::Suld3DI32Clamp:
3870b57cec5SDimitry Andric   case NVPTXISD::Suld3DI64Clamp:
3880b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I8Clamp:
3890b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I16Clamp:
3900b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I32Clamp:
3910b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I64Clamp:
3920b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I8Clamp:
3930b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I16Clamp:
3940b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I32Clamp:
3950b57cec5SDimitry Andric   case NVPTXISD::Suld1DI8Trap:
3960b57cec5SDimitry Andric   case NVPTXISD::Suld1DI16Trap:
3970b57cec5SDimitry Andric   case NVPTXISD::Suld1DI32Trap:
3980b57cec5SDimitry Andric   case NVPTXISD::Suld1DI64Trap:
3990b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I8Trap:
4000b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I16Trap:
4010b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I32Trap:
4020b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I64Trap:
4030b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I8Trap:
4040b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I16Trap:
4050b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I32Trap:
4060b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI8Trap:
4070b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI16Trap:
4080b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI32Trap:
4090b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI64Trap:
4100b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I8Trap:
4110b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I16Trap:
4120b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I32Trap:
4130b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I64Trap:
4140b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I8Trap:
4150b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I16Trap:
4160b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I32Trap:
4170b57cec5SDimitry Andric   case NVPTXISD::Suld2DI8Trap:
4180b57cec5SDimitry Andric   case NVPTXISD::Suld2DI16Trap:
4190b57cec5SDimitry Andric   case NVPTXISD::Suld2DI32Trap:
4200b57cec5SDimitry Andric   case NVPTXISD::Suld2DI64Trap:
4210b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I8Trap:
4220b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I16Trap:
4230b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I32Trap:
4240b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I64Trap:
4250b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I8Trap:
4260b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I16Trap:
4270b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I32Trap:
4280b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI8Trap:
4290b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI16Trap:
4300b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI32Trap:
4310b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI64Trap:
4320b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I8Trap:
4330b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I16Trap:
4340b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I32Trap:
4350b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I64Trap:
4360b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I8Trap:
4370b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I16Trap:
4380b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I32Trap:
4390b57cec5SDimitry Andric   case NVPTXISD::Suld3DI8Trap:
4400b57cec5SDimitry Andric   case NVPTXISD::Suld3DI16Trap:
4410b57cec5SDimitry Andric   case NVPTXISD::Suld3DI32Trap:
4420b57cec5SDimitry Andric   case NVPTXISD::Suld3DI64Trap:
4430b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I8Trap:
4440b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I16Trap:
4450b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I32Trap:
4460b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I64Trap:
4470b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I8Trap:
4480b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I16Trap:
4490b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I32Trap:
4500b57cec5SDimitry Andric   case NVPTXISD::Suld1DI8Zero:
4510b57cec5SDimitry Andric   case NVPTXISD::Suld1DI16Zero:
4520b57cec5SDimitry Andric   case NVPTXISD::Suld1DI32Zero:
4530b57cec5SDimitry Andric   case NVPTXISD::Suld1DI64Zero:
4540b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I8Zero:
4550b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I16Zero:
4560b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I32Zero:
4570b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I64Zero:
4580b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I8Zero:
4590b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I16Zero:
4600b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I32Zero:
4610b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI8Zero:
4620b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI16Zero:
4630b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI32Zero:
4640b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI64Zero:
4650b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I8Zero:
4660b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I16Zero:
4670b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I32Zero:
4680b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I64Zero:
4690b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I8Zero:
4700b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I16Zero:
4710b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I32Zero:
4720b57cec5SDimitry Andric   case NVPTXISD::Suld2DI8Zero:
4730b57cec5SDimitry Andric   case NVPTXISD::Suld2DI16Zero:
4740b57cec5SDimitry Andric   case NVPTXISD::Suld2DI32Zero:
4750b57cec5SDimitry Andric   case NVPTXISD::Suld2DI64Zero:
4760b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I8Zero:
4770b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I16Zero:
4780b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I32Zero:
4790b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I64Zero:
4800b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I8Zero:
4810b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I16Zero:
4820b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I32Zero:
4830b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI8Zero:
4840b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI16Zero:
4850b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI32Zero:
4860b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI64Zero:
4870b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I8Zero:
4880b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I16Zero:
4890b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I32Zero:
4900b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I64Zero:
4910b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I8Zero:
4920b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I16Zero:
4930b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I32Zero:
4940b57cec5SDimitry Andric   case NVPTXISD::Suld3DI8Zero:
4950b57cec5SDimitry Andric   case NVPTXISD::Suld3DI16Zero:
4960b57cec5SDimitry Andric   case NVPTXISD::Suld3DI32Zero:
4970b57cec5SDimitry Andric   case NVPTXISD::Suld3DI64Zero:
4980b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I8Zero:
4990b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I16Zero:
5000b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I32Zero:
5010b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I64Zero:
5020b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I8Zero:
5030b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I16Zero:
5040b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I32Zero:
5050b57cec5SDimitry Andric     if (trySurfaceIntrinsic(N))
5060b57cec5SDimitry Andric       return;
5070b57cec5SDimitry Andric     break;
5080b57cec5SDimitry Andric   case ISD::AND:
5090b57cec5SDimitry Andric   case ISD::SRA:
5100b57cec5SDimitry Andric   case ISD::SRL:
5110b57cec5SDimitry Andric     // Try to select BFE
5120b57cec5SDimitry Andric     if (tryBFE(N))
5130b57cec5SDimitry Andric       return;
5140b57cec5SDimitry Andric     break;
5150b57cec5SDimitry Andric   case ISD::ADDRSPACECAST:
5160b57cec5SDimitry Andric     SelectAddrSpaceCast(N);
5170b57cec5SDimitry Andric     return;
5180b57cec5SDimitry Andric   case ISD::ConstantFP:
51906c3fb27SDimitry Andric     if (tryConstantFP(N))
5200b57cec5SDimitry Andric       return;
5210b57cec5SDimitry Andric     break;
522*0fca6ea1SDimitry Andric   case ISD::CopyToReg: {
523*0fca6ea1SDimitry Andric     if (N->getOperand(1).getValueType() == MVT::i128) {
524*0fca6ea1SDimitry Andric       SelectV2I64toI128(N);
525*0fca6ea1SDimitry Andric       return;
526*0fca6ea1SDimitry Andric     }
527*0fca6ea1SDimitry Andric     break;
528*0fca6ea1SDimitry Andric   }
529*0fca6ea1SDimitry Andric   case ISD::CopyFromReg: {
530*0fca6ea1SDimitry Andric     if (N->getOperand(1).getValueType() == MVT::i128) {
531*0fca6ea1SDimitry Andric       SelectI128toV2I64(N);
532*0fca6ea1SDimitry Andric       return;
533*0fca6ea1SDimitry Andric     }
534*0fca6ea1SDimitry Andric     break;
535*0fca6ea1SDimitry Andric   }
5360b57cec5SDimitry Andric   default:
5370b57cec5SDimitry Andric     break;
5380b57cec5SDimitry Andric   }
5390b57cec5SDimitry Andric   SelectCode(N);
5400b57cec5SDimitry Andric }
5410b57cec5SDimitry Andric 
5420b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryIntrinsicChain(SDNode *N) {
543647cbc5dSDimitry Andric   unsigned IID = N->getConstantOperandVal(1);
5440b57cec5SDimitry Andric   switch (IID) {
5450b57cec5SDimitry Andric   default:
5460b57cec5SDimitry Andric     return false;
5470b57cec5SDimitry Andric   case Intrinsic::nvvm_ldg_global_f:
5480b57cec5SDimitry Andric   case Intrinsic::nvvm_ldg_global_i:
5490b57cec5SDimitry Andric   case Intrinsic::nvvm_ldg_global_p:
5500b57cec5SDimitry Andric   case Intrinsic::nvvm_ldu_global_f:
5510b57cec5SDimitry Andric   case Intrinsic::nvvm_ldu_global_i:
5520b57cec5SDimitry Andric   case Intrinsic::nvvm_ldu_global_p:
5530b57cec5SDimitry Andric     return tryLDGLDU(N);
5540b57cec5SDimitry Andric   }
5550b57cec5SDimitry Andric }
5560b57cec5SDimitry Andric 
55706c3fb27SDimitry Andric // There's no way to specify FP16 and BF16 immediates in .(b)f16 ops, so we
55806c3fb27SDimitry Andric // have to load them into an .(b)f16 register first.
55906c3fb27SDimitry Andric bool NVPTXDAGToDAGISel::tryConstantFP(SDNode *N) {
56006c3fb27SDimitry Andric   if (N->getValueType(0) != MVT::f16 && N->getValueType(0) != MVT::bf16)
5610b57cec5SDimitry Andric     return false;
5620b57cec5SDimitry Andric   SDValue Val = CurDAG->getTargetConstantFP(
56306c3fb27SDimitry Andric       cast<ConstantFPSDNode>(N)->getValueAPF(), SDLoc(N), N->getValueType(0));
56406c3fb27SDimitry Andric   SDNode *LoadConstF16 = CurDAG->getMachineNode(
56506c3fb27SDimitry Andric       (N->getValueType(0) == MVT::f16 ? NVPTX::LOAD_CONST_F16
56606c3fb27SDimitry Andric                                       : NVPTX::LOAD_CONST_BF16),
56706c3fb27SDimitry Andric       SDLoc(N), N->getValueType(0), Val);
5680b57cec5SDimitry Andric   ReplaceNode(N, LoadConstF16);
5690b57cec5SDimitry Andric   return true;
5700b57cec5SDimitry Andric }
5710b57cec5SDimitry Andric 
5720b57cec5SDimitry Andric // Map ISD:CONDCODE value to appropriate CmpMode expected by
5730b57cec5SDimitry Andric // NVPTXInstPrinter::printCmpMode()
5740b57cec5SDimitry Andric static unsigned getPTXCmpMode(const CondCodeSDNode &CondCode, bool FTZ) {
5750b57cec5SDimitry Andric   using NVPTX::PTXCmpMode::CmpMode;
5760b57cec5SDimitry Andric   unsigned PTXCmpMode = [](ISD::CondCode CC) {
5770b57cec5SDimitry Andric     switch (CC) {
5780b57cec5SDimitry Andric     default:
5790b57cec5SDimitry Andric       llvm_unreachable("Unexpected condition code.");
5800b57cec5SDimitry Andric     case ISD::SETOEQ:
5810b57cec5SDimitry Andric       return CmpMode::EQ;
5820b57cec5SDimitry Andric     case ISD::SETOGT:
5830b57cec5SDimitry Andric       return CmpMode::GT;
5840b57cec5SDimitry Andric     case ISD::SETOGE:
5850b57cec5SDimitry Andric       return CmpMode::GE;
5860b57cec5SDimitry Andric     case ISD::SETOLT:
5870b57cec5SDimitry Andric       return CmpMode::LT;
5880b57cec5SDimitry Andric     case ISD::SETOLE:
5890b57cec5SDimitry Andric       return CmpMode::LE;
5900b57cec5SDimitry Andric     case ISD::SETONE:
5910b57cec5SDimitry Andric       return CmpMode::NE;
5920b57cec5SDimitry Andric     case ISD::SETO:
5930b57cec5SDimitry Andric       return CmpMode::NUM;
5940b57cec5SDimitry Andric     case ISD::SETUO:
5950b57cec5SDimitry Andric       return CmpMode::NotANumber;
5960b57cec5SDimitry Andric     case ISD::SETUEQ:
5970b57cec5SDimitry Andric       return CmpMode::EQU;
5980b57cec5SDimitry Andric     case ISD::SETUGT:
5990b57cec5SDimitry Andric       return CmpMode::GTU;
6000b57cec5SDimitry Andric     case ISD::SETUGE:
6010b57cec5SDimitry Andric       return CmpMode::GEU;
6020b57cec5SDimitry Andric     case ISD::SETULT:
6030b57cec5SDimitry Andric       return CmpMode::LTU;
6040b57cec5SDimitry Andric     case ISD::SETULE:
6050b57cec5SDimitry Andric       return CmpMode::LEU;
6060b57cec5SDimitry Andric     case ISD::SETUNE:
6070b57cec5SDimitry Andric       return CmpMode::NEU;
6080b57cec5SDimitry Andric     case ISD::SETEQ:
6090b57cec5SDimitry Andric       return CmpMode::EQ;
6100b57cec5SDimitry Andric     case ISD::SETGT:
6110b57cec5SDimitry Andric       return CmpMode::GT;
6120b57cec5SDimitry Andric     case ISD::SETGE:
6130b57cec5SDimitry Andric       return CmpMode::GE;
6140b57cec5SDimitry Andric     case ISD::SETLT:
6150b57cec5SDimitry Andric       return CmpMode::LT;
6160b57cec5SDimitry Andric     case ISD::SETLE:
6170b57cec5SDimitry Andric       return CmpMode::LE;
6180b57cec5SDimitry Andric     case ISD::SETNE:
6190b57cec5SDimitry Andric       return CmpMode::NE;
6200b57cec5SDimitry Andric     }
6210b57cec5SDimitry Andric   }(CondCode.get());
6220b57cec5SDimitry Andric 
6230b57cec5SDimitry Andric   if (FTZ)
6240b57cec5SDimitry Andric     PTXCmpMode |= NVPTX::PTXCmpMode::FTZ_FLAG;
6250b57cec5SDimitry Andric 
6260b57cec5SDimitry Andric   return PTXCmpMode;
6270b57cec5SDimitry Andric }
6280b57cec5SDimitry Andric 
6290b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectSETP_F16X2(SDNode *N) {
6300b57cec5SDimitry Andric   unsigned PTXCmpMode =
6310b57cec5SDimitry Andric       getPTXCmpMode(*cast<CondCodeSDNode>(N->getOperand(2)), useF32FTZ());
6320b57cec5SDimitry Andric   SDLoc DL(N);
6330b57cec5SDimitry Andric   SDNode *SetP = CurDAG->getMachineNode(
6340b57cec5SDimitry Andric       NVPTX::SETP_f16x2rr, DL, MVT::i1, MVT::i1, N->getOperand(0),
6350b57cec5SDimitry Andric       N->getOperand(1), CurDAG->getTargetConstant(PTXCmpMode, DL, MVT::i32));
6360b57cec5SDimitry Andric   ReplaceNode(N, SetP);
6370b57cec5SDimitry Andric   return true;
6380b57cec5SDimitry Andric }
6390b57cec5SDimitry Andric 
6405f757f3fSDimitry Andric bool NVPTXDAGToDAGISel::SelectSETP_BF16X2(SDNode *N) {
6415f757f3fSDimitry Andric   unsigned PTXCmpMode =
6425f757f3fSDimitry Andric       getPTXCmpMode(*cast<CondCodeSDNode>(N->getOperand(2)), useF32FTZ());
6435f757f3fSDimitry Andric   SDLoc DL(N);
6445f757f3fSDimitry Andric   SDNode *SetP = CurDAG->getMachineNode(
6455f757f3fSDimitry Andric       NVPTX::SETP_bf16x2rr, DL, MVT::i1, MVT::i1, N->getOperand(0),
6465f757f3fSDimitry Andric       N->getOperand(1), CurDAG->getTargetConstant(PTXCmpMode, DL, MVT::i32));
6475f757f3fSDimitry Andric   ReplaceNode(N, SetP);
6485f757f3fSDimitry Andric   return true;
6495f757f3fSDimitry Andric }
6505f757f3fSDimitry Andric 
6510b57cec5SDimitry Andric // Find all instances of extract_vector_elt that use this v2f16 vector
6520b57cec5SDimitry Andric // and coalesce them into a scattering move instruction.
6530b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryEXTRACT_VECTOR_ELEMENT(SDNode *N) {
6540b57cec5SDimitry Andric   SDValue Vector = N->getOperand(0);
6550b57cec5SDimitry Andric 
6565f757f3fSDimitry Andric   // We only care about 16x2 as it's the only real vector type we
6570b57cec5SDimitry Andric   // need to deal with.
65806c3fb27SDimitry Andric   MVT VT = Vector.getSimpleValueType();
6595f757f3fSDimitry Andric   if (!Isv2x16VT(VT))
6600b57cec5SDimitry Andric     return false;
6610b57cec5SDimitry Andric   // Find and record all uses of this vector that extract element 0 or 1.
6620b57cec5SDimitry Andric   SmallVector<SDNode *, 4> E0, E1;
663bdd1243dSDimitry Andric   for (auto *U : Vector.getNode()->uses()) {
6640b57cec5SDimitry Andric     if (U->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6650b57cec5SDimitry Andric       continue;
6660b57cec5SDimitry Andric     if (U->getOperand(0) != Vector)
6670b57cec5SDimitry Andric       continue;
6680b57cec5SDimitry Andric     if (const ConstantSDNode *IdxConst =
6690b57cec5SDimitry Andric             dyn_cast<ConstantSDNode>(U->getOperand(1))) {
6700b57cec5SDimitry Andric       if (IdxConst->getZExtValue() == 0)
6710b57cec5SDimitry Andric         E0.push_back(U);
6720b57cec5SDimitry Andric       else if (IdxConst->getZExtValue() == 1)
6730b57cec5SDimitry Andric         E1.push_back(U);
6740b57cec5SDimitry Andric       else
6750b57cec5SDimitry Andric         llvm_unreachable("Invalid vector index.");
6760b57cec5SDimitry Andric     }
6770b57cec5SDimitry Andric   }
6780b57cec5SDimitry Andric 
6790b57cec5SDimitry Andric   // There's no point scattering f16x2 if we only ever access one
6800b57cec5SDimitry Andric   // element of it.
6810b57cec5SDimitry Andric   if (E0.empty() || E1.empty())
6820b57cec5SDimitry Andric     return false;
6830b57cec5SDimitry Andric 
6840b57cec5SDimitry Andric   // Merge (f16 extractelt(V, 0), f16 extractelt(V,1))
6850b57cec5SDimitry Andric   // into f16,f16 SplitF16x2(V)
68606c3fb27SDimitry Andric   MVT EltVT = VT.getVectorElementType();
6870b57cec5SDimitry Andric   SDNode *ScatterOp =
68806c3fb27SDimitry Andric       CurDAG->getMachineNode(NVPTX::I32toV2I16, SDLoc(N), EltVT, EltVT, Vector);
6890b57cec5SDimitry Andric   for (auto *Node : E0)
6900b57cec5SDimitry Andric     ReplaceUses(SDValue(Node, 0), SDValue(ScatterOp, 0));
6910b57cec5SDimitry Andric   for (auto *Node : E1)
6920b57cec5SDimitry Andric     ReplaceUses(SDValue(Node, 0), SDValue(ScatterOp, 1));
6930b57cec5SDimitry Andric 
6940b57cec5SDimitry Andric   return true;
6950b57cec5SDimitry Andric }
6960b57cec5SDimitry Andric 
6970b57cec5SDimitry Andric static unsigned int getCodeAddrSpace(MemSDNode *N) {
6980b57cec5SDimitry Andric   const Value *Src = N->getMemOperand()->getValue();
6990b57cec5SDimitry Andric 
7000b57cec5SDimitry Andric   if (!Src)
7010b57cec5SDimitry Andric     return NVPTX::PTXLdStInstCode::GENERIC;
7020b57cec5SDimitry Andric 
7030b57cec5SDimitry Andric   if (auto *PT = dyn_cast<PointerType>(Src->getType())) {
7040b57cec5SDimitry Andric     switch (PT->getAddressSpace()) {
7050b57cec5SDimitry Andric     case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
7060b57cec5SDimitry Andric     case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
7070b57cec5SDimitry Andric     case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
7080b57cec5SDimitry Andric     case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
7090b57cec5SDimitry Andric     case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
7100b57cec5SDimitry Andric     case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
7110b57cec5SDimitry Andric     default: break;
7120b57cec5SDimitry Andric     }
7130b57cec5SDimitry Andric   }
7140b57cec5SDimitry Andric   return NVPTX::PTXLdStInstCode::GENERIC;
7150b57cec5SDimitry Andric }
7160b57cec5SDimitry Andric 
7170b57cec5SDimitry Andric static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget,
7180b57cec5SDimitry Andric                           unsigned CodeAddrSpace, MachineFunction *F) {
7190b57cec5SDimitry Andric   // We use ldg (i.e. ld.global.nc) for invariant loads from the global address
7200b57cec5SDimitry Andric   // space.
7210b57cec5SDimitry Andric   //
7220b57cec5SDimitry Andric   // We have two ways of identifying invariant loads: Loads may be explicitly
7230b57cec5SDimitry Andric   // marked as invariant, or we may infer them to be invariant.
7240b57cec5SDimitry Andric   //
7250b57cec5SDimitry Andric   // We currently infer invariance for loads from
7260b57cec5SDimitry Andric   //  - constant global variables, and
7270b57cec5SDimitry Andric   //  - kernel function pointer params that are noalias (i.e. __restrict) and
7280b57cec5SDimitry Andric   //    never written to.
7290b57cec5SDimitry Andric   //
7300b57cec5SDimitry Andric   // TODO: Perform a more powerful invariance analysis (ideally IPO, and ideally
7310b57cec5SDimitry Andric   // not during the SelectionDAG phase).
7320b57cec5SDimitry Andric   //
7330b57cec5SDimitry Andric   // TODO: Infer invariance only at -O2.  We still want to use ldg at -O0 for
7340b57cec5SDimitry Andric   // explicitly invariant loads because these are how clang tells us to use ldg
7350b57cec5SDimitry Andric   // when the user uses a builtin.
7360b57cec5SDimitry Andric   if (!Subtarget.hasLDG() || CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL)
7370b57cec5SDimitry Andric     return false;
7380b57cec5SDimitry Andric 
7390b57cec5SDimitry Andric   if (N->isInvariant())
7400b57cec5SDimitry Andric     return true;
7410b57cec5SDimitry Andric 
7420b57cec5SDimitry Andric   bool IsKernelFn = isKernelFunction(F->getFunction());
7430b57cec5SDimitry Andric 
744e8d8bef9SDimitry Andric   // We use getUnderlyingObjects() here instead of getUnderlyingObject() mainly
7450b57cec5SDimitry Andric   // because the former looks through phi nodes while the latter does not. We
7460b57cec5SDimitry Andric   // need to look through phi nodes to handle pointer induction variables.
7470b57cec5SDimitry Andric   SmallVector<const Value *, 8> Objs;
748e8d8bef9SDimitry Andric   getUnderlyingObjects(N->getMemOperand()->getValue(), Objs);
7490b57cec5SDimitry Andric 
7500b57cec5SDimitry Andric   return all_of(Objs, [&](const Value *V) {
7510b57cec5SDimitry Andric     if (auto *A = dyn_cast<const Argument>(V))
7520b57cec5SDimitry Andric       return IsKernelFn && A->onlyReadsMemory() && A->hasNoAliasAttr();
7530b57cec5SDimitry Andric     if (auto *GV = dyn_cast<const GlobalVariable>(V))
7540b57cec5SDimitry Andric       return GV->isConstant();
7550b57cec5SDimitry Andric     return false;
7560b57cec5SDimitry Andric   });
7570b57cec5SDimitry Andric }
7580b57cec5SDimitry Andric 
7590b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryIntrinsicNoChain(SDNode *N) {
760647cbc5dSDimitry Andric   unsigned IID = N->getConstantOperandVal(0);
7610b57cec5SDimitry Andric   switch (IID) {
7620b57cec5SDimitry Andric   default:
7630b57cec5SDimitry Andric     return false;
7640b57cec5SDimitry Andric   case Intrinsic::nvvm_texsurf_handle_internal:
7650b57cec5SDimitry Andric     SelectTexSurfHandle(N);
7660b57cec5SDimitry Andric     return true;
7670b57cec5SDimitry Andric   }
7680b57cec5SDimitry Andric }
7690b57cec5SDimitry Andric 
7700b57cec5SDimitry Andric void NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
7710b57cec5SDimitry Andric   // Op 0 is the intrinsic ID
7720b57cec5SDimitry Andric   SDValue Wrapper = N->getOperand(1);
7730b57cec5SDimitry Andric   SDValue GlobalVal = Wrapper.getOperand(0);
7740b57cec5SDimitry Andric   ReplaceNode(N, CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N),
7750b57cec5SDimitry Andric                                         MVT::i64, GlobalVal));
7760b57cec5SDimitry Andric }
7770b57cec5SDimitry Andric 
7780b57cec5SDimitry Andric void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
7790b57cec5SDimitry Andric   SDValue Src = N->getOperand(0);
7800b57cec5SDimitry Andric   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
7810b57cec5SDimitry Andric   unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
7820b57cec5SDimitry Andric   unsigned DstAddrSpace = CastN->getDestAddressSpace();
7830b57cec5SDimitry Andric   assert(SrcAddrSpace != DstAddrSpace &&
7840b57cec5SDimitry Andric          "addrspacecast must be between different address spaces");
7850b57cec5SDimitry Andric 
7860b57cec5SDimitry Andric   if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
7870b57cec5SDimitry Andric     // Specific to generic
7880b57cec5SDimitry Andric     unsigned Opc;
7890b57cec5SDimitry Andric     switch (SrcAddrSpace) {
7900b57cec5SDimitry Andric     default: report_fatal_error("Bad address space in addrspacecast");
7910b57cec5SDimitry Andric     case ADDRESS_SPACE_GLOBAL:
792*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? NVPTX::cvta_global_64 : NVPTX::cvta_global;
7930b57cec5SDimitry Andric       break;
7940b57cec5SDimitry Andric     case ADDRESS_SPACE_SHARED:
795*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? (TM.getPointerSizeInBits(SrcAddrSpace) == 32
796*0fca6ea1SDimitry Andric                                 ? NVPTX::cvta_shared_6432
797*0fca6ea1SDimitry Andric                                 : NVPTX::cvta_shared_64)
798*0fca6ea1SDimitry Andric                          : NVPTX::cvta_shared;
7990b57cec5SDimitry Andric       break;
8000b57cec5SDimitry Andric     case ADDRESS_SPACE_CONST:
801*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? (TM.getPointerSizeInBits(SrcAddrSpace) == 32
802*0fca6ea1SDimitry Andric                                 ? NVPTX::cvta_const_6432
803*0fca6ea1SDimitry Andric                                 : NVPTX::cvta_const_64)
804*0fca6ea1SDimitry Andric                          : NVPTX::cvta_const;
8050b57cec5SDimitry Andric       break;
8060b57cec5SDimitry Andric     case ADDRESS_SPACE_LOCAL:
807*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? (TM.getPointerSizeInBits(SrcAddrSpace) == 32
808*0fca6ea1SDimitry Andric                                 ? NVPTX::cvta_local_6432
809*0fca6ea1SDimitry Andric                                 : NVPTX::cvta_local_64)
810*0fca6ea1SDimitry Andric                          : NVPTX::cvta_local;
8110b57cec5SDimitry Andric       break;
8120b57cec5SDimitry Andric     }
8130b57cec5SDimitry Andric     ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
8140b57cec5SDimitry Andric                                           Src));
8150b57cec5SDimitry Andric     return;
8160b57cec5SDimitry Andric   } else {
8170b57cec5SDimitry Andric     // Generic to specific
8180b57cec5SDimitry Andric     if (SrcAddrSpace != 0)
8190b57cec5SDimitry Andric       report_fatal_error("Cannot cast between two non-generic address spaces");
8200b57cec5SDimitry Andric     unsigned Opc;
8210b57cec5SDimitry Andric     switch (DstAddrSpace) {
8220b57cec5SDimitry Andric     default: report_fatal_error("Bad address space in addrspacecast");
8230b57cec5SDimitry Andric     case ADDRESS_SPACE_GLOBAL:
824*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? NVPTX::cvta_to_global_64 : NVPTX::cvta_to_global;
8250b57cec5SDimitry Andric       break;
8260b57cec5SDimitry Andric     case ADDRESS_SPACE_SHARED:
827*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? (TM.getPointerSizeInBits(DstAddrSpace) == 32
828*0fca6ea1SDimitry Andric                                 ? NVPTX::cvta_to_shared_3264
829*0fca6ea1SDimitry Andric                                 : NVPTX::cvta_to_shared_64)
830*0fca6ea1SDimitry Andric                          : NVPTX::cvta_to_shared;
8310b57cec5SDimitry Andric       break;
8320b57cec5SDimitry Andric     case ADDRESS_SPACE_CONST:
833*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? (TM.getPointerSizeInBits(DstAddrSpace) == 32
834*0fca6ea1SDimitry Andric                                 ? NVPTX::cvta_to_const_3264
835*0fca6ea1SDimitry Andric                                 : NVPTX::cvta_to_const_64)
836*0fca6ea1SDimitry Andric                          : NVPTX::cvta_to_const;
8370b57cec5SDimitry Andric       break;
8380b57cec5SDimitry Andric     case ADDRESS_SPACE_LOCAL:
839*0fca6ea1SDimitry Andric       Opc = TM.is64Bit() ? (TM.getPointerSizeInBits(DstAddrSpace) == 32
840*0fca6ea1SDimitry Andric                                 ? NVPTX::cvta_to_local_3264
841*0fca6ea1SDimitry Andric                                 : NVPTX::cvta_to_local_64)
842*0fca6ea1SDimitry Andric                          : NVPTX::cvta_to_local;
8430b57cec5SDimitry Andric       break;
8440b57cec5SDimitry Andric     case ADDRESS_SPACE_PARAM:
8450b57cec5SDimitry Andric       Opc = TM.is64Bit() ? NVPTX::nvvm_ptr_gen_to_param_64
8460b57cec5SDimitry Andric                          : NVPTX::nvvm_ptr_gen_to_param;
8470b57cec5SDimitry Andric       break;
8480b57cec5SDimitry Andric     }
8490b57cec5SDimitry Andric     ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
8500b57cec5SDimitry Andric                                           Src));
8510b57cec5SDimitry Andric     return;
8520b57cec5SDimitry Andric   }
8530b57cec5SDimitry Andric }
8540b57cec5SDimitry Andric 
8550b57cec5SDimitry Andric // Helper function template to reduce amount of boilerplate code for
8560b57cec5SDimitry Andric // opcode selection.
857bdd1243dSDimitry Andric static std::optional<unsigned>
858bdd1243dSDimitry Andric pickOpcodeForVT(MVT::SimpleValueType VT, unsigned Opcode_i8,
859bdd1243dSDimitry Andric                 unsigned Opcode_i16, unsigned Opcode_i32,
86006c3fb27SDimitry Andric                 std::optional<unsigned> Opcode_i64, unsigned Opcode_f32,
861bdd1243dSDimitry Andric                 std::optional<unsigned> Opcode_f64) {
8620b57cec5SDimitry Andric   switch (VT) {
8630b57cec5SDimitry Andric   case MVT::i1:
8640b57cec5SDimitry Andric   case MVT::i8:
8650b57cec5SDimitry Andric     return Opcode_i8;
8660b57cec5SDimitry Andric   case MVT::i16:
8670b57cec5SDimitry Andric     return Opcode_i16;
8680b57cec5SDimitry Andric   case MVT::i32:
8690b57cec5SDimitry Andric     return Opcode_i32;
8700b57cec5SDimitry Andric   case MVT::i64:
8710b57cec5SDimitry Andric     return Opcode_i64;
8720b57cec5SDimitry Andric   case MVT::f16:
873bdd1243dSDimitry Andric   case MVT::bf16:
87406c3fb27SDimitry Andric     return Opcode_i16;
8750b57cec5SDimitry Andric   case MVT::v2f16:
876bdd1243dSDimitry Andric   case MVT::v2bf16:
8775f757f3fSDimitry Andric   case MVT::v2i16:
8785f757f3fSDimitry Andric   case MVT::v4i8:
87906c3fb27SDimitry Andric     return Opcode_i32;
8800b57cec5SDimitry Andric   case MVT::f32:
8810b57cec5SDimitry Andric     return Opcode_f32;
8820b57cec5SDimitry Andric   case MVT::f64:
8830b57cec5SDimitry Andric     return Opcode_f64;
8840b57cec5SDimitry Andric   default:
885bdd1243dSDimitry Andric     return std::nullopt;
8860b57cec5SDimitry Andric   }
8870b57cec5SDimitry Andric }
8880b57cec5SDimitry Andric 
889bdd1243dSDimitry Andric static int getLdStRegType(EVT VT) {
890bdd1243dSDimitry Andric   if (VT.isFloatingPoint())
891bdd1243dSDimitry Andric     switch (VT.getSimpleVT().SimpleTy) {
892bdd1243dSDimitry Andric     case MVT::f16:
893bdd1243dSDimitry Andric     case MVT::bf16:
894bdd1243dSDimitry Andric     case MVT::v2f16:
895bdd1243dSDimitry Andric     case MVT::v2bf16:
896bdd1243dSDimitry Andric       return NVPTX::PTXLdStInstCode::Untyped;
897bdd1243dSDimitry Andric     default:
898bdd1243dSDimitry Andric       return NVPTX::PTXLdStInstCode::Float;
899bdd1243dSDimitry Andric     }
900bdd1243dSDimitry Andric   else
901bdd1243dSDimitry Andric     return NVPTX::PTXLdStInstCode::Unsigned;
902bdd1243dSDimitry Andric }
903bdd1243dSDimitry Andric 
9040b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
9050b57cec5SDimitry Andric   SDLoc dl(N);
9060b57cec5SDimitry Andric   MemSDNode *LD = cast<MemSDNode>(N);
9070b57cec5SDimitry Andric   assert(LD->readMem() && "Expected load");
9080b57cec5SDimitry Andric   LoadSDNode *PlainLoad = dyn_cast<LoadSDNode>(N);
9090b57cec5SDimitry Andric   EVT LoadedVT = LD->getMemoryVT();
9100b57cec5SDimitry Andric   SDNode *NVPTXLD = nullptr;
9110b57cec5SDimitry Andric 
9120b57cec5SDimitry Andric   // do not support pre/post inc/dec
9130b57cec5SDimitry Andric   if (PlainLoad && PlainLoad->isIndexed())
9140b57cec5SDimitry Andric     return false;
9150b57cec5SDimitry Andric 
9160b57cec5SDimitry Andric   if (!LoadedVT.isSimple())
9170b57cec5SDimitry Andric     return false;
9180b57cec5SDimitry Andric 
919fe6060f1SDimitry Andric   AtomicOrdering Ordering = LD->getSuccessOrdering();
9200b57cec5SDimitry Andric   // In order to lower atomic loads with stronger guarantees we would need to
9210b57cec5SDimitry Andric   // use load.acquire or insert fences. However these features were only added
9220b57cec5SDimitry Andric   // with PTX ISA 6.0 / sm_70.
9230b57cec5SDimitry Andric   // TODO: Check if we can actually use the new instructions and implement them.
9240b57cec5SDimitry Andric   if (isStrongerThanMonotonic(Ordering))
9250b57cec5SDimitry Andric     return false;
9260b57cec5SDimitry Andric 
9270b57cec5SDimitry Andric   // Address Space Setting
9280b57cec5SDimitry Andric   unsigned int CodeAddrSpace = getCodeAddrSpace(LD);
9290b57cec5SDimitry Andric   if (canLowerToLDG(LD, *Subtarget, CodeAddrSpace, MF)) {
9300b57cec5SDimitry Andric     return tryLDGLDU(N);
9310b57cec5SDimitry Andric   }
9320b57cec5SDimitry Andric 
9330b57cec5SDimitry Andric   unsigned int PointerSize =
9340b57cec5SDimitry Andric       CurDAG->getDataLayout().getPointerSizeInBits(LD->getAddressSpace());
9350b57cec5SDimitry Andric 
9360b57cec5SDimitry Andric   // Volatile Setting
9370b57cec5SDimitry Andric   // - .volatile is only available for .global and .shared
9380b57cec5SDimitry Andric   // - .volatile has the same memory synchronization semantics as .relaxed.sys
9390b57cec5SDimitry Andric   bool isVolatile = LD->isVolatile() || Ordering == AtomicOrdering::Monotonic;
9400b57cec5SDimitry Andric   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
9410b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
9420b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
9430b57cec5SDimitry Andric     isVolatile = false;
9440b57cec5SDimitry Andric 
9450b57cec5SDimitry Andric   // Type Setting: fromType + fromTypeWidth
9460b57cec5SDimitry Andric   //
9470b57cec5SDimitry Andric   // Sign   : ISD::SEXTLOAD
9480b57cec5SDimitry Andric   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
9490b57cec5SDimitry Andric   //          type is integer
9500b57cec5SDimitry Andric   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
9510b57cec5SDimitry Andric   MVT SimpleVT = LoadedVT.getSimpleVT();
9520b57cec5SDimitry Andric   MVT ScalarVT = SimpleVT.getScalarType();
9530b57cec5SDimitry Andric   // Read at least 8 bits (predicates are stored as 8-bit values)
954480093f4SDimitry Andric   unsigned fromTypeWidth = std::max(8U, (unsigned)ScalarVT.getSizeInBits());
9550b57cec5SDimitry Andric   unsigned int fromType;
9560b57cec5SDimitry Andric 
9570b57cec5SDimitry Andric   // Vector Setting
9580b57cec5SDimitry Andric   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
9590b57cec5SDimitry Andric   if (SimpleVT.isVector()) {
9605f757f3fSDimitry Andric     assert((Isv2x16VT(LoadedVT) || LoadedVT == MVT::v4i8) &&
961bdd1243dSDimitry Andric            "Unexpected vector type");
9625f757f3fSDimitry Andric     // v2f16/v2bf16/v2i16 is loaded using ld.b32
9630b57cec5SDimitry Andric     fromTypeWidth = 32;
9640b57cec5SDimitry Andric   }
9650b57cec5SDimitry Andric 
9660b57cec5SDimitry Andric   if (PlainLoad && (PlainLoad->getExtensionType() == ISD::SEXTLOAD))
9670b57cec5SDimitry Andric     fromType = NVPTX::PTXLdStInstCode::Signed;
9680b57cec5SDimitry Andric   else
969bdd1243dSDimitry Andric     fromType = getLdStRegType(ScalarVT);
9700b57cec5SDimitry Andric 
9710b57cec5SDimitry Andric   // Create the machine instruction DAG
9720b57cec5SDimitry Andric   SDValue Chain = N->getOperand(0);
9730b57cec5SDimitry Andric   SDValue N1 = N->getOperand(1);
9740b57cec5SDimitry Andric   SDValue Addr;
9750b57cec5SDimitry Andric   SDValue Offset, Base;
976bdd1243dSDimitry Andric   std::optional<unsigned> Opcode;
9770b57cec5SDimitry Andric   MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
9780b57cec5SDimitry Andric 
9790b57cec5SDimitry Andric   if (SelectDirectAddr(N1, Addr)) {
98006c3fb27SDimitry Andric     Opcode = pickOpcodeForVT(TargetVT, NVPTX::LD_i8_avar, NVPTX::LD_i16_avar,
98106c3fb27SDimitry Andric                              NVPTX::LD_i32_avar, NVPTX::LD_i64_avar,
9820b57cec5SDimitry Andric                              NVPTX::LD_f32_avar, NVPTX::LD_f64_avar);
9830b57cec5SDimitry Andric     if (!Opcode)
9840b57cec5SDimitry Andric       return false;
9850b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
9860b57cec5SDimitry Andric                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
9870b57cec5SDimitry Andric                       getI32Imm(fromTypeWidth, dl), Addr, Chain };
98881ad6265SDimitry Andric     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
9890b57cec5SDimitry Andric   } else if (PointerSize == 64 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
9900b57cec5SDimitry Andric                                : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
9910b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(TargetVT, NVPTX::LD_i8_asi, NVPTX::LD_i16_asi,
9920b57cec5SDimitry Andric                              NVPTX::LD_i32_asi, NVPTX::LD_i64_asi,
9930b57cec5SDimitry Andric                              NVPTX::LD_f32_asi, NVPTX::LD_f64_asi);
9940b57cec5SDimitry Andric     if (!Opcode)
9950b57cec5SDimitry Andric       return false;
9960b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
9970b57cec5SDimitry Andric                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
9980b57cec5SDimitry Andric                       getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
99981ad6265SDimitry Andric     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
10000b57cec5SDimitry Andric   } else if (PointerSize == 64 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
10010b57cec5SDimitry Andric                                : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
10020b57cec5SDimitry Andric     if (PointerSize == 64)
100306c3fb27SDimitry Andric       Opcode =
100406c3fb27SDimitry Andric           pickOpcodeForVT(TargetVT, NVPTX::LD_i8_ari_64, NVPTX::LD_i16_ari_64,
100506c3fb27SDimitry Andric                           NVPTX::LD_i32_ari_64, NVPTX::LD_i64_ari_64,
100606c3fb27SDimitry Andric                           NVPTX::LD_f32_ari_64, NVPTX::LD_f64_ari_64);
10070b57cec5SDimitry Andric     else
100806c3fb27SDimitry Andric       Opcode = pickOpcodeForVT(TargetVT, NVPTX::LD_i8_ari, NVPTX::LD_i16_ari,
100906c3fb27SDimitry Andric                                NVPTX::LD_i32_ari, NVPTX::LD_i64_ari,
10100b57cec5SDimitry Andric                                NVPTX::LD_f32_ari, NVPTX::LD_f64_ari);
10110b57cec5SDimitry Andric     if (!Opcode)
10120b57cec5SDimitry Andric       return false;
10130b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
10140b57cec5SDimitry Andric                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
10150b57cec5SDimitry Andric                       getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
101681ad6265SDimitry Andric     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
10170b57cec5SDimitry Andric   } else {
10180b57cec5SDimitry Andric     if (PointerSize == 64)
101906c3fb27SDimitry Andric       Opcode =
102006c3fb27SDimitry Andric           pickOpcodeForVT(TargetVT, NVPTX::LD_i8_areg_64, NVPTX::LD_i16_areg_64,
102106c3fb27SDimitry Andric                           NVPTX::LD_i32_areg_64, NVPTX::LD_i64_areg_64,
102206c3fb27SDimitry Andric                           NVPTX::LD_f32_areg_64, NVPTX::LD_f64_areg_64);
10230b57cec5SDimitry Andric     else
102406c3fb27SDimitry Andric       Opcode = pickOpcodeForVT(TargetVT, NVPTX::LD_i8_areg, NVPTX::LD_i16_areg,
102506c3fb27SDimitry Andric                                NVPTX::LD_i32_areg, NVPTX::LD_i64_areg,
10260b57cec5SDimitry Andric                                NVPTX::LD_f32_areg, NVPTX::LD_f64_areg);
10270b57cec5SDimitry Andric     if (!Opcode)
10280b57cec5SDimitry Andric       return false;
10290b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(CodeAddrSpace, dl),
10300b57cec5SDimitry Andric                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
10310b57cec5SDimitry Andric                       getI32Imm(fromTypeWidth, dl), N1, Chain };
103281ad6265SDimitry Andric     NVPTXLD = CurDAG->getMachineNode(*Opcode, dl, TargetVT, MVT::Other, Ops);
10330b57cec5SDimitry Andric   }
10340b57cec5SDimitry Andric 
10350b57cec5SDimitry Andric   if (!NVPTXLD)
10360b57cec5SDimitry Andric     return false;
10370b57cec5SDimitry Andric 
10380b57cec5SDimitry Andric   MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
10390b57cec5SDimitry Andric   CurDAG->setNodeMemRefs(cast<MachineSDNode>(NVPTXLD), {MemRef});
10400b57cec5SDimitry Andric 
10410b57cec5SDimitry Andric   ReplaceNode(N, NVPTXLD);
10420b57cec5SDimitry Andric   return true;
10430b57cec5SDimitry Andric }
10440b57cec5SDimitry Andric 
10450b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
10460b57cec5SDimitry Andric 
10470b57cec5SDimitry Andric   SDValue Chain = N->getOperand(0);
10480b57cec5SDimitry Andric   SDValue Op1 = N->getOperand(1);
10490b57cec5SDimitry Andric   SDValue Addr, Offset, Base;
1050bdd1243dSDimitry Andric   std::optional<unsigned> Opcode;
10510b57cec5SDimitry Andric   SDLoc DL(N);
10520b57cec5SDimitry Andric   SDNode *LD;
10530b57cec5SDimitry Andric   MemSDNode *MemSD = cast<MemSDNode>(N);
10540b57cec5SDimitry Andric   EVT LoadedVT = MemSD->getMemoryVT();
10550b57cec5SDimitry Andric 
10560b57cec5SDimitry Andric   if (!LoadedVT.isSimple())
10570b57cec5SDimitry Andric     return false;
10580b57cec5SDimitry Andric 
10590b57cec5SDimitry Andric   // Address Space Setting
10600b57cec5SDimitry Andric   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
10610b57cec5SDimitry Andric   if (canLowerToLDG(MemSD, *Subtarget, CodeAddrSpace, MF)) {
10620b57cec5SDimitry Andric     return tryLDGLDU(N);
10630b57cec5SDimitry Andric   }
10640b57cec5SDimitry Andric 
10650b57cec5SDimitry Andric   unsigned int PointerSize =
10660b57cec5SDimitry Andric       CurDAG->getDataLayout().getPointerSizeInBits(MemSD->getAddressSpace());
10670b57cec5SDimitry Andric 
10680b57cec5SDimitry Andric   // Volatile Setting
10690b57cec5SDimitry Andric   // - .volatile is only availalble for .global and .shared
10700b57cec5SDimitry Andric   bool IsVolatile = MemSD->isVolatile();
10710b57cec5SDimitry Andric   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
10720b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
10730b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
10740b57cec5SDimitry Andric     IsVolatile = false;
10750b57cec5SDimitry Andric 
10760b57cec5SDimitry Andric   // Vector Setting
10770b57cec5SDimitry Andric   MVT SimpleVT = LoadedVT.getSimpleVT();
10780b57cec5SDimitry Andric 
10790b57cec5SDimitry Andric   // Type Setting: fromType + fromTypeWidth
10800b57cec5SDimitry Andric   //
10810b57cec5SDimitry Andric   // Sign   : ISD::SEXTLOAD
10820b57cec5SDimitry Andric   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
10830b57cec5SDimitry Andric   //          type is integer
10840b57cec5SDimitry Andric   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
10850b57cec5SDimitry Andric   MVT ScalarVT = SimpleVT.getScalarType();
10860b57cec5SDimitry Andric   // Read at least 8 bits (predicates are stored as 8-bit values)
1087480093f4SDimitry Andric   unsigned FromTypeWidth = std::max(8U, (unsigned)ScalarVT.getSizeInBits());
10880b57cec5SDimitry Andric   unsigned int FromType;
10890b57cec5SDimitry Andric   // The last operand holds the original LoadSDNode::getExtensionType() value
10900b57cec5SDimitry Andric   unsigned ExtensionType = cast<ConstantSDNode>(
10910b57cec5SDimitry Andric       N->getOperand(N->getNumOperands() - 1))->getZExtValue();
10920b57cec5SDimitry Andric   if (ExtensionType == ISD::SEXTLOAD)
10930b57cec5SDimitry Andric     FromType = NVPTX::PTXLdStInstCode::Signed;
10940b57cec5SDimitry Andric   else
1095bdd1243dSDimitry Andric     FromType = getLdStRegType(ScalarVT);
10960b57cec5SDimitry Andric 
10970b57cec5SDimitry Andric   unsigned VecType;
10980b57cec5SDimitry Andric 
10990b57cec5SDimitry Andric   switch (N->getOpcode()) {
11000b57cec5SDimitry Andric   case NVPTXISD::LoadV2:
11010b57cec5SDimitry Andric     VecType = NVPTX::PTXLdStInstCode::V2;
11020b57cec5SDimitry Andric     break;
11030b57cec5SDimitry Andric   case NVPTXISD::LoadV4:
11040b57cec5SDimitry Andric     VecType = NVPTX::PTXLdStInstCode::V4;
11050b57cec5SDimitry Andric     break;
11060b57cec5SDimitry Andric   default:
11070b57cec5SDimitry Andric     return false;
11080b57cec5SDimitry Andric   }
11090b57cec5SDimitry Andric 
11100b57cec5SDimitry Andric   EVT EltVT = N->getValueType(0);
11110b57cec5SDimitry Andric 
11125f757f3fSDimitry Andric   // v8x16 is a special case. PTX doesn't have ld.v8.16
11135f757f3fSDimitry Andric   // instruction. Instead, we split the vector into v2x16 chunks and
11140b57cec5SDimitry Andric   // load them with ld.v4.b32.
11155f757f3fSDimitry Andric   if (Isv2x16VT(EltVT)) {
11160b57cec5SDimitry Andric     assert(N->getOpcode() == NVPTXISD::LoadV4 && "Unexpected load opcode.");
11170b57cec5SDimitry Andric     EltVT = MVT::i32;
11180b57cec5SDimitry Andric     FromType = NVPTX::PTXLdStInstCode::Untyped;
11190b57cec5SDimitry Andric     FromTypeWidth = 32;
11200b57cec5SDimitry Andric   }
11210b57cec5SDimitry Andric 
11220b57cec5SDimitry Andric   if (SelectDirectAddr(Op1, Addr)) {
11230b57cec5SDimitry Andric     switch (N->getOpcode()) {
11240b57cec5SDimitry Andric     default:
11250b57cec5SDimitry Andric       return false;
11260b57cec5SDimitry Andric     case NVPTXISD::LoadV2:
11270b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
11280b57cec5SDimitry Andric                                NVPTX::LDV_i8_v2_avar, NVPTX::LDV_i16_v2_avar,
11290b57cec5SDimitry Andric                                NVPTX::LDV_i32_v2_avar, NVPTX::LDV_i64_v2_avar,
11300b57cec5SDimitry Andric                                NVPTX::LDV_f32_v2_avar, NVPTX::LDV_f64_v2_avar);
11310b57cec5SDimitry Andric       break;
11320b57cec5SDimitry Andric     case NVPTXISD::LoadV4:
113306c3fb27SDimitry Andric       Opcode =
113406c3fb27SDimitry Andric           pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_avar,
113506c3fb27SDimitry Andric                           NVPTX::LDV_i16_v4_avar, NVPTX::LDV_i32_v4_avar,
113606c3fb27SDimitry Andric                           std::nullopt, NVPTX::LDV_f32_v4_avar, std::nullopt);
11370b57cec5SDimitry Andric       break;
11380b57cec5SDimitry Andric     }
11390b57cec5SDimitry Andric     if (!Opcode)
11400b57cec5SDimitry Andric       return false;
11410b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
11420b57cec5SDimitry Andric                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
11430b57cec5SDimitry Andric                       getI32Imm(FromTypeWidth, DL), Addr, Chain };
114481ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
11450b57cec5SDimitry Andric   } else if (PointerSize == 64
11460b57cec5SDimitry Andric                  ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
11470b57cec5SDimitry Andric                  : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
11480b57cec5SDimitry Andric     switch (N->getOpcode()) {
11490b57cec5SDimitry Andric     default:
11500b57cec5SDimitry Andric       return false;
11510b57cec5SDimitry Andric     case NVPTXISD::LoadV2:
11520b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
11530b57cec5SDimitry Andric                                NVPTX::LDV_i8_v2_asi, NVPTX::LDV_i16_v2_asi,
11540b57cec5SDimitry Andric                                NVPTX::LDV_i32_v2_asi, NVPTX::LDV_i64_v2_asi,
11550b57cec5SDimitry Andric                                NVPTX::LDV_f32_v2_asi, NVPTX::LDV_f64_v2_asi);
11560b57cec5SDimitry Andric       break;
11570b57cec5SDimitry Andric     case NVPTXISD::LoadV4:
115806c3fb27SDimitry Andric       Opcode =
115906c3fb27SDimitry Andric           pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_asi,
116006c3fb27SDimitry Andric                           NVPTX::LDV_i16_v4_asi, NVPTX::LDV_i32_v4_asi,
116106c3fb27SDimitry Andric                           std::nullopt, NVPTX::LDV_f32_v4_asi, std::nullopt);
11620b57cec5SDimitry Andric       break;
11630b57cec5SDimitry Andric     }
11640b57cec5SDimitry Andric     if (!Opcode)
11650b57cec5SDimitry Andric       return false;
11660b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
11670b57cec5SDimitry Andric                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
11680b57cec5SDimitry Andric                       getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
116981ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
11700b57cec5SDimitry Andric   } else if (PointerSize == 64
11710b57cec5SDimitry Andric                  ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
11720b57cec5SDimitry Andric                  : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
11730b57cec5SDimitry Andric     if (PointerSize == 64) {
11740b57cec5SDimitry Andric       switch (N->getOpcode()) {
11750b57cec5SDimitry Andric       default:
11760b57cec5SDimitry Andric         return false;
11770b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
117806c3fb27SDimitry Andric         Opcode =
117906c3fb27SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
118006c3fb27SDimitry Andric                             NVPTX::LDV_i8_v2_ari_64, NVPTX::LDV_i16_v2_ari_64,
118106c3fb27SDimitry Andric                             NVPTX::LDV_i32_v2_ari_64, NVPTX::LDV_i64_v2_ari_64,
118206c3fb27SDimitry Andric                             NVPTX::LDV_f32_v2_ari_64, NVPTX::LDV_f64_v2_ari_64);
11830b57cec5SDimitry Andric         break;
11840b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
11850b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(
11860b57cec5SDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_ari_64,
1187bdd1243dSDimitry Andric             NVPTX::LDV_i16_v4_ari_64, NVPTX::LDV_i32_v4_ari_64, std::nullopt,
1188bdd1243dSDimitry Andric             NVPTX::LDV_f32_v4_ari_64, std::nullopt);
11890b57cec5SDimitry Andric         break;
11900b57cec5SDimitry Andric       }
11910b57cec5SDimitry Andric     } else {
11920b57cec5SDimitry Andric       switch (N->getOpcode()) {
11930b57cec5SDimitry Andric       default:
11940b57cec5SDimitry Andric         return false;
11950b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
11960b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
11970b57cec5SDimitry Andric                                  NVPTX::LDV_i8_v2_ari, NVPTX::LDV_i16_v2_ari,
11980b57cec5SDimitry Andric                                  NVPTX::LDV_i32_v2_ari, NVPTX::LDV_i64_v2_ari,
11990b57cec5SDimitry Andric                                  NVPTX::LDV_f32_v2_ari, NVPTX::LDV_f64_v2_ari);
12000b57cec5SDimitry Andric         break;
12010b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
120206c3fb27SDimitry Andric         Opcode =
120306c3fb27SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_ari,
120406c3fb27SDimitry Andric                             NVPTX::LDV_i16_v4_ari, NVPTX::LDV_i32_v4_ari,
120506c3fb27SDimitry Andric                             std::nullopt, NVPTX::LDV_f32_v4_ari, std::nullopt);
12060b57cec5SDimitry Andric         break;
12070b57cec5SDimitry Andric       }
12080b57cec5SDimitry Andric     }
12090b57cec5SDimitry Andric     if (!Opcode)
12100b57cec5SDimitry Andric       return false;
12110b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
12120b57cec5SDimitry Andric                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
12130b57cec5SDimitry Andric                       getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
12140b57cec5SDimitry Andric 
121581ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
12160b57cec5SDimitry Andric   } else {
12170b57cec5SDimitry Andric     if (PointerSize == 64) {
12180b57cec5SDimitry Andric       switch (N->getOpcode()) {
12190b57cec5SDimitry Andric       default:
12200b57cec5SDimitry Andric         return false;
12210b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
12220b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(
12230b57cec5SDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v2_areg_64,
12240b57cec5SDimitry Andric             NVPTX::LDV_i16_v2_areg_64, NVPTX::LDV_i32_v2_areg_64,
122506c3fb27SDimitry Andric             NVPTX::LDV_i64_v2_areg_64, NVPTX::LDV_f32_v2_areg_64,
12260b57cec5SDimitry Andric             NVPTX::LDV_f64_v2_areg_64);
12270b57cec5SDimitry Andric         break;
12280b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
12290b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(
12300b57cec5SDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_areg_64,
1231bdd1243dSDimitry Andric             NVPTX::LDV_i16_v4_areg_64, NVPTX::LDV_i32_v4_areg_64, std::nullopt,
1232bdd1243dSDimitry Andric             NVPTX::LDV_f32_v4_areg_64, std::nullopt);
12330b57cec5SDimitry Andric         break;
12340b57cec5SDimitry Andric       }
12350b57cec5SDimitry Andric     } else {
12360b57cec5SDimitry Andric       switch (N->getOpcode()) {
12370b57cec5SDimitry Andric       default:
12380b57cec5SDimitry Andric         return false;
12390b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
12400b57cec5SDimitry Andric         Opcode =
12410b57cec5SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v2_areg,
12420b57cec5SDimitry Andric                             NVPTX::LDV_i16_v2_areg, NVPTX::LDV_i32_v2_areg,
124306c3fb27SDimitry Andric                             NVPTX::LDV_i64_v2_areg, NVPTX::LDV_f32_v2_areg,
12440b57cec5SDimitry Andric                             NVPTX::LDV_f64_v2_areg);
12450b57cec5SDimitry Andric         break;
12460b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
124706c3fb27SDimitry Andric         Opcode =
124806c3fb27SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_areg,
124906c3fb27SDimitry Andric                             NVPTX::LDV_i16_v4_areg, NVPTX::LDV_i32_v4_areg,
125006c3fb27SDimitry Andric                             std::nullopt, NVPTX::LDV_f32_v4_areg, std::nullopt);
12510b57cec5SDimitry Andric         break;
12520b57cec5SDimitry Andric       }
12530b57cec5SDimitry Andric     }
12540b57cec5SDimitry Andric     if (!Opcode)
12550b57cec5SDimitry Andric       return false;
12560b57cec5SDimitry Andric     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
12570b57cec5SDimitry Andric                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
12580b57cec5SDimitry Andric                       getI32Imm(FromTypeWidth, DL), Op1, Chain };
125981ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, N->getVTList(), Ops);
12600b57cec5SDimitry Andric   }
12610b57cec5SDimitry Andric 
12620b57cec5SDimitry Andric   MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
12630b57cec5SDimitry Andric   CurDAG->setNodeMemRefs(cast<MachineSDNode>(LD), {MemRef});
12640b57cec5SDimitry Andric 
12650b57cec5SDimitry Andric   ReplaceNode(N, LD);
12660b57cec5SDimitry Andric   return true;
12670b57cec5SDimitry Andric }
12680b57cec5SDimitry Andric 
12690b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
12700b57cec5SDimitry Andric 
12710b57cec5SDimitry Andric   SDValue Chain = N->getOperand(0);
12720b57cec5SDimitry Andric   SDValue Op1;
12730b57cec5SDimitry Andric   MemSDNode *Mem;
12740b57cec5SDimitry Andric   bool IsLDG = true;
12750b57cec5SDimitry Andric 
12760b57cec5SDimitry Andric   // If this is an LDG intrinsic, the address is the third operand. If its an
12770b57cec5SDimitry Andric   // LDG/LDU SD node (from custom vector handling), then its the second operand
12780b57cec5SDimitry Andric   if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
12790b57cec5SDimitry Andric     Op1 = N->getOperand(2);
12800b57cec5SDimitry Andric     Mem = cast<MemIntrinsicSDNode>(N);
1281647cbc5dSDimitry Andric     unsigned IID = N->getConstantOperandVal(1);
12820b57cec5SDimitry Andric     switch (IID) {
12830b57cec5SDimitry Andric     default:
12840b57cec5SDimitry Andric       return false;
12850b57cec5SDimitry Andric     case Intrinsic::nvvm_ldg_global_f:
12860b57cec5SDimitry Andric     case Intrinsic::nvvm_ldg_global_i:
12870b57cec5SDimitry Andric     case Intrinsic::nvvm_ldg_global_p:
12880b57cec5SDimitry Andric       IsLDG = true;
12890b57cec5SDimitry Andric       break;
12900b57cec5SDimitry Andric     case Intrinsic::nvvm_ldu_global_f:
12910b57cec5SDimitry Andric     case Intrinsic::nvvm_ldu_global_i:
12920b57cec5SDimitry Andric     case Intrinsic::nvvm_ldu_global_p:
12930b57cec5SDimitry Andric       IsLDG = false;
12940b57cec5SDimitry Andric       break;
12950b57cec5SDimitry Andric     }
12960b57cec5SDimitry Andric   } else {
12970b57cec5SDimitry Andric     Op1 = N->getOperand(1);
12980b57cec5SDimitry Andric     Mem = cast<MemSDNode>(N);
12990b57cec5SDimitry Andric   }
13000b57cec5SDimitry Andric 
1301bdd1243dSDimitry Andric   std::optional<unsigned> Opcode;
13020b57cec5SDimitry Andric   SDLoc DL(N);
13030b57cec5SDimitry Andric   SDNode *LD;
13040b57cec5SDimitry Andric   SDValue Base, Offset, Addr;
13055f757f3fSDimitry Andric   EVT OrigType = N->getValueType(0);
13060b57cec5SDimitry Andric 
13070b57cec5SDimitry Andric   EVT EltVT = Mem->getMemoryVT();
13080b57cec5SDimitry Andric   unsigned NumElts = 1;
13090b57cec5SDimitry Andric   if (EltVT.isVector()) {
13100b57cec5SDimitry Andric     NumElts = EltVT.getVectorNumElements();
13110b57cec5SDimitry Andric     EltVT = EltVT.getVectorElementType();
13125f757f3fSDimitry Andric     // vectors of 16bits type are loaded/stored as multiples of v2x16 elements.
13135f757f3fSDimitry Andric     if ((EltVT == MVT::f16 && OrigType == MVT::v2f16) ||
13145f757f3fSDimitry Andric         (EltVT == MVT::bf16 && OrigType == MVT::v2bf16) ||
13155f757f3fSDimitry Andric         (EltVT == MVT::i16 && OrigType == MVT::v2i16)) {
13160b57cec5SDimitry Andric       assert(NumElts % 2 == 0 && "Vector must have even number of elements");
13175f757f3fSDimitry Andric       EltVT = OrigType;
13180b57cec5SDimitry Andric       NumElts /= 2;
13195f757f3fSDimitry Andric     } else if (OrigType == MVT::v4i8) {
13205f757f3fSDimitry Andric       EltVT = OrigType;
13215f757f3fSDimitry Andric       NumElts = 1;
13220b57cec5SDimitry Andric     }
13230b57cec5SDimitry Andric   }
13240b57cec5SDimitry Andric 
13250b57cec5SDimitry Andric   // Build the "promoted" result VTList for the load. If we are really loading
13260b57cec5SDimitry Andric   // i8s, then the return type will be promoted to i16 since we do not expose
13270b57cec5SDimitry Andric   // 8-bit registers in NVPTX.
13280b57cec5SDimitry Andric   EVT NodeVT = (EltVT == MVT::i8) ? MVT::i16 : EltVT;
13290b57cec5SDimitry Andric   SmallVector<EVT, 5> InstVTs;
13300b57cec5SDimitry Andric   for (unsigned i = 0; i != NumElts; ++i) {
13310b57cec5SDimitry Andric     InstVTs.push_back(NodeVT);
13320b57cec5SDimitry Andric   }
13330b57cec5SDimitry Andric   InstVTs.push_back(MVT::Other);
13340b57cec5SDimitry Andric   SDVTList InstVTList = CurDAG->getVTList(InstVTs);
13350b57cec5SDimitry Andric 
13360b57cec5SDimitry Andric   if (SelectDirectAddr(Op1, Addr)) {
13370b57cec5SDimitry Andric     switch (N->getOpcode()) {
13380b57cec5SDimitry Andric     default:
13390b57cec5SDimitry Andric       return false;
13400b57cec5SDimitry Andric     case ISD::LOAD:
13410b57cec5SDimitry Andric     case ISD::INTRINSIC_W_CHAIN:
13420b57cec5SDimitry Andric       if (IsLDG)
13430b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
13440b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_GLOBAL_i8avar,
13450b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_GLOBAL_i16avar,
13460b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_GLOBAL_i32avar,
13470b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_GLOBAL_i64avar,
13480b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_GLOBAL_f32avar,
13490b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_GLOBAL_f64avar);
13500b57cec5SDimitry Andric       else
13510b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
13520b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_GLOBAL_i8avar,
13530b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_GLOBAL_i16avar,
13540b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_GLOBAL_i32avar,
13550b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_GLOBAL_i64avar,
13560b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_GLOBAL_f32avar,
13570b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_GLOBAL_f64avar);
13580b57cec5SDimitry Andric       break;
13590b57cec5SDimitry Andric     case NVPTXISD::LoadV2:
13600b57cec5SDimitry Andric     case NVPTXISD::LDGV2:
13610b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
13620b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar,
13630b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar,
13640b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar,
13650b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar,
13660b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar,
13670b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar);
13680b57cec5SDimitry Andric       break;
13690b57cec5SDimitry Andric     case NVPTXISD::LDUV2:
13700b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
13710b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar,
13720b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar,
13730b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar,
13740b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar,
13750b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar,
13760b57cec5SDimitry Andric                                NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar);
13770b57cec5SDimitry Andric       break;
13780b57cec5SDimitry Andric     case NVPTXISD::LoadV4:
13790b57cec5SDimitry Andric     case NVPTXISD::LDGV4:
1380bdd1243dSDimitry Andric       Opcode = pickOpcodeForVT(
1381bdd1243dSDimitry Andric           EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar,
13820b57cec5SDimitry Andric           NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar,
1383bdd1243dSDimitry Andric           NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar, std::nullopt,
1384bdd1243dSDimitry Andric           NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar, std::nullopt);
13850b57cec5SDimitry Andric       break;
13860b57cec5SDimitry Andric     case NVPTXISD::LDUV4:
1387bdd1243dSDimitry Andric       Opcode = pickOpcodeForVT(
1388bdd1243dSDimitry Andric           EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar,
13890b57cec5SDimitry Andric           NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar,
1390bdd1243dSDimitry Andric           NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar, std::nullopt,
1391bdd1243dSDimitry Andric           NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar, std::nullopt);
13920b57cec5SDimitry Andric       break;
13930b57cec5SDimitry Andric     }
13940b57cec5SDimitry Andric     if (!Opcode)
13950b57cec5SDimitry Andric       return false;
13960b57cec5SDimitry Andric     SDValue Ops[] = { Addr, Chain };
139781ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, InstVTList, Ops);
13980b57cec5SDimitry Andric   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
13990b57cec5SDimitry Andric                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
14000b57cec5SDimitry Andric     if (TM.is64Bit()) {
14010b57cec5SDimitry Andric       switch (N->getOpcode()) {
14020b57cec5SDimitry Andric       default:
14030b57cec5SDimitry Andric         return false;
14040b57cec5SDimitry Andric       case ISD::LOAD:
14050b57cec5SDimitry Andric       case ISD::INTRINSIC_W_CHAIN:
14060b57cec5SDimitry Andric         if (IsLDG)
14070b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14080b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i8ari64,
14090b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i16ari64,
14100b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i32ari64,
14110b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i64ari64,
14120b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_f32ari64,
14130b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_f64ari64);
14140b57cec5SDimitry Andric         else
14150b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14160b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i8ari64,
14170b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i16ari64,
14180b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i32ari64,
14190b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i64ari64,
14200b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_f32ari64,
14210b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_f64ari64);
14220b57cec5SDimitry Andric         break;
14230b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
14240b57cec5SDimitry Andric       case NVPTXISD::LDGV2:
14250b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14260b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64,
14270b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64,
14280b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64,
14290b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64,
14300b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64,
14310b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64);
14320b57cec5SDimitry Andric         break;
14330b57cec5SDimitry Andric       case NVPTXISD::LDUV2:
14340b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14350b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64,
14360b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64,
14370b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64,
14380b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64,
14390b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64,
14400b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64);
14410b57cec5SDimitry Andric         break;
14420b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
14430b57cec5SDimitry Andric       case NVPTXISD::LDGV4:
1444bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1445bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64,
14460b57cec5SDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64,
1447bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64, std::nullopt,
1448bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64, std::nullopt);
14490b57cec5SDimitry Andric         break;
14500b57cec5SDimitry Andric       case NVPTXISD::LDUV4:
1451bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1452bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64,
14530b57cec5SDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64,
1454bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64, std::nullopt,
1455bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64, std::nullopt);
14560b57cec5SDimitry Andric         break;
14570b57cec5SDimitry Andric       }
14580b57cec5SDimitry Andric     } else {
14590b57cec5SDimitry Andric       switch (N->getOpcode()) {
14600b57cec5SDimitry Andric       default:
14610b57cec5SDimitry Andric         return false;
14620b57cec5SDimitry Andric       case ISD::LOAD:
14630b57cec5SDimitry Andric       case ISD::INTRINSIC_W_CHAIN:
14640b57cec5SDimitry Andric         if (IsLDG)
14650b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14660b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i8ari,
14670b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i16ari,
14680b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i32ari,
14690b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i64ari,
14700b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_f32ari,
14710b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_f64ari);
14720b57cec5SDimitry Andric         else
14730b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14740b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i8ari,
14750b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i16ari,
14760b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i32ari,
14770b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i64ari,
14780b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_f32ari,
14790b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_f64ari);
14800b57cec5SDimitry Andric         break;
14810b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
14820b57cec5SDimitry Andric       case NVPTXISD::LDGV2:
14830b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14840b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32,
14850b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32,
14860b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32,
14870b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32,
14880b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32,
14890b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32);
14900b57cec5SDimitry Andric         break;
14910b57cec5SDimitry Andric       case NVPTXISD::LDUV2:
14920b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
14930b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32,
14940b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32,
14950b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32,
14960b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32,
14970b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32,
14980b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32);
14990b57cec5SDimitry Andric         break;
15000b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
15010b57cec5SDimitry Andric       case NVPTXISD::LDGV4:
1502bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1503bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32,
15040b57cec5SDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32,
1505bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32, std::nullopt,
1506bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32, std::nullopt);
15070b57cec5SDimitry Andric         break;
15080b57cec5SDimitry Andric       case NVPTXISD::LDUV4:
1509bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1510bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32,
15110b57cec5SDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32,
1512bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32, std::nullopt,
1513bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32, std::nullopt);
15140b57cec5SDimitry Andric         break;
15150b57cec5SDimitry Andric       }
15160b57cec5SDimitry Andric     }
15170b57cec5SDimitry Andric     if (!Opcode)
15180b57cec5SDimitry Andric       return false;
15190b57cec5SDimitry Andric     SDValue Ops[] = {Base, Offset, Chain};
152081ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, InstVTList, Ops);
15210b57cec5SDimitry Andric   } else {
15220b57cec5SDimitry Andric     if (TM.is64Bit()) {
15230b57cec5SDimitry Andric       switch (N->getOpcode()) {
15240b57cec5SDimitry Andric       default:
15250b57cec5SDimitry Andric         return false;
15260b57cec5SDimitry Andric       case ISD::LOAD:
15270b57cec5SDimitry Andric       case ISD::INTRINSIC_W_CHAIN:
15280b57cec5SDimitry Andric         if (IsLDG)
15290b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
15300b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i8areg64,
15310b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i16areg64,
15320b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i32areg64,
15330b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_i64areg64,
15340b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_f32areg64,
15350b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDG_GLOBAL_f64areg64);
15360b57cec5SDimitry Andric         else
15370b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
15380b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i8areg64,
15390b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i16areg64,
15400b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i32areg64,
15410b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_i64areg64,
15420b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_f32areg64,
15430b57cec5SDimitry Andric                                        NVPTX::INT_PTX_LDU_GLOBAL_f64areg64);
15440b57cec5SDimitry Andric         break;
15450b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
15460b57cec5SDimitry Andric       case NVPTXISD::LDGV2:
15470b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
15480b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64,
15490b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64,
15500b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64,
15510b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64,
15520b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64,
15530b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64);
15540b57cec5SDimitry Andric         break;
15550b57cec5SDimitry Andric       case NVPTXISD::LDUV2:
15560b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
15570b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64,
15580b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64,
15590b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64,
15600b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64,
15610b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64,
15620b57cec5SDimitry Andric                                      NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64);
15630b57cec5SDimitry Andric         break;
15640b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
15650b57cec5SDimitry Andric       case NVPTXISD::LDGV4:
1566bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1567bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64,
15680b57cec5SDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64,
1569bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64, std::nullopt,
1570bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64, std::nullopt);
15710b57cec5SDimitry Andric         break;
15720b57cec5SDimitry Andric       case NVPTXISD::LDUV4:
1573bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1574bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64,
15750b57cec5SDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64,
1576bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64, std::nullopt,
1577bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64, std::nullopt);
15780b57cec5SDimitry Andric         break;
15790b57cec5SDimitry Andric       }
15800b57cec5SDimitry Andric     } else {
15810b57cec5SDimitry Andric       switch (N->getOpcode()) {
15820b57cec5SDimitry Andric       default:
15830b57cec5SDimitry Andric         return false;
15840b57cec5SDimitry Andric       case ISD::LOAD:
15850b57cec5SDimitry Andric       case ISD::INTRINSIC_W_CHAIN:
15860b57cec5SDimitry Andric         if (IsLDG)
15870b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
15880b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i8areg,
15890b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i16areg,
15900b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i32areg,
15910b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_i64areg,
15920b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_f32areg,
15930b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDG_GLOBAL_f64areg);
15940b57cec5SDimitry Andric         else
15950b57cec5SDimitry Andric           Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
15960b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i8areg,
15970b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i16areg,
15980b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i32areg,
15990b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_i64areg,
16000b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_f32areg,
16010b57cec5SDimitry Andric                                    NVPTX::INT_PTX_LDU_GLOBAL_f64areg);
16020b57cec5SDimitry Andric         break;
16030b57cec5SDimitry Andric       case NVPTXISD::LoadV2:
16040b57cec5SDimitry Andric       case NVPTXISD::LDGV2:
16050b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
16060b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32,
16070b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32,
16080b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32,
16090b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32,
16100b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32,
16110b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32);
16120b57cec5SDimitry Andric         break;
16130b57cec5SDimitry Andric       case NVPTXISD::LDUV2:
16140b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
16150b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32,
16160b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32,
16170b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32,
16180b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32,
16190b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32,
16200b57cec5SDimitry Andric                                  NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32);
16210b57cec5SDimitry Andric         break;
16220b57cec5SDimitry Andric       case NVPTXISD::LoadV4:
16230b57cec5SDimitry Andric       case NVPTXISD::LDGV4:
1624bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1625bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32,
16260b57cec5SDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32,
1627bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32, std::nullopt,
1628bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32, std::nullopt);
16290b57cec5SDimitry Andric         break;
16300b57cec5SDimitry Andric       case NVPTXISD::LDUV4:
1631bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(
1632bdd1243dSDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32,
16330b57cec5SDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32,
1634bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32, std::nullopt,
1635bdd1243dSDimitry Andric             NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32, std::nullopt);
16360b57cec5SDimitry Andric         break;
16370b57cec5SDimitry Andric       }
16380b57cec5SDimitry Andric     }
16390b57cec5SDimitry Andric     if (!Opcode)
16400b57cec5SDimitry Andric       return false;
16410b57cec5SDimitry Andric     SDValue Ops[] = { Op1, Chain };
164281ad6265SDimitry Andric     LD = CurDAG->getMachineNode(*Opcode, DL, InstVTList, Ops);
16430b57cec5SDimitry Andric   }
16440b57cec5SDimitry Andric 
16450b57cec5SDimitry Andric   // For automatic generation of LDG (through SelectLoad[Vector], not the
16460b57cec5SDimitry Andric   // intrinsics), we may have an extending load like:
16470b57cec5SDimitry Andric   //
16480b57cec5SDimitry Andric   //   i32,ch = load<LD1[%data1(addrspace=1)], zext from i8> t0, t7, undef:i64
16490b57cec5SDimitry Andric   //
16500b57cec5SDimitry Andric   // In this case, the matching logic above will select a load for the original
16510b57cec5SDimitry Andric   // memory type (in this case, i8) and our types will not match (the node needs
16520b57cec5SDimitry Andric   // to return an i32 in this case). Our LDG/LDU nodes do not support the
16530b57cec5SDimitry Andric   // concept of sign-/zero-extension, so emulate it here by adding an explicit
16540b57cec5SDimitry Andric   // CVT instruction. Ptxas should clean up any redundancies here.
16550b57cec5SDimitry Andric 
16560b57cec5SDimitry Andric   LoadSDNode *LdNode = dyn_cast<LoadSDNode>(N);
16570b57cec5SDimitry Andric 
165806c3fb27SDimitry Andric   if (OrigType != EltVT &&
165906c3fb27SDimitry Andric       (LdNode || (OrigType.isFloatingPoint() && EltVT.isFloatingPoint()))) {
16600b57cec5SDimitry Andric     // We have an extending-load. The instruction we selected operates on the
16610b57cec5SDimitry Andric     // smaller type, but the SDNode we are replacing has the larger type. We
16620b57cec5SDimitry Andric     // need to emit a CVT to make the types match.
166306c3fb27SDimitry Andric     unsigned CvtOpc =
166406c3fb27SDimitry Andric         GetConvertOpcode(OrigType.getSimpleVT(), EltVT.getSimpleVT(), LdNode);
16650b57cec5SDimitry Andric 
16660b57cec5SDimitry Andric     // For each output value, apply the manual sign/zero-extension and make sure
16670b57cec5SDimitry Andric     // all users of the load go through that CVT.
16680b57cec5SDimitry Andric     for (unsigned i = 0; i != NumElts; ++i) {
16690b57cec5SDimitry Andric       SDValue Res(LD, i);
16700b57cec5SDimitry Andric       SDValue OrigVal(N, i);
16710b57cec5SDimitry Andric 
16720b57cec5SDimitry Andric       SDNode *CvtNode =
16730b57cec5SDimitry Andric         CurDAG->getMachineNode(CvtOpc, DL, OrigType, Res,
16740b57cec5SDimitry Andric                                CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
16750b57cec5SDimitry Andric                                                          DL, MVT::i32));
16760b57cec5SDimitry Andric       ReplaceUses(OrigVal, SDValue(CvtNode, 0));
16770b57cec5SDimitry Andric     }
16780b57cec5SDimitry Andric   }
16790b57cec5SDimitry Andric 
16800b57cec5SDimitry Andric   ReplaceNode(N, LD);
16810b57cec5SDimitry Andric   return true;
16820b57cec5SDimitry Andric }
16830b57cec5SDimitry Andric 
16840b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
16850b57cec5SDimitry Andric   SDLoc dl(N);
16860b57cec5SDimitry Andric   MemSDNode *ST = cast<MemSDNode>(N);
16870b57cec5SDimitry Andric   assert(ST->writeMem() && "Expected store");
16880b57cec5SDimitry Andric   StoreSDNode *PlainStore = dyn_cast<StoreSDNode>(N);
16890b57cec5SDimitry Andric   AtomicSDNode *AtomicStore = dyn_cast<AtomicSDNode>(N);
16900b57cec5SDimitry Andric   assert((PlainStore || AtomicStore) && "Expected store");
16910b57cec5SDimitry Andric   EVT StoreVT = ST->getMemoryVT();
16920b57cec5SDimitry Andric   SDNode *NVPTXST = nullptr;
16930b57cec5SDimitry Andric 
16940b57cec5SDimitry Andric   // do not support pre/post inc/dec
16950b57cec5SDimitry Andric   if (PlainStore && PlainStore->isIndexed())
16960b57cec5SDimitry Andric     return false;
16970b57cec5SDimitry Andric 
16980b57cec5SDimitry Andric   if (!StoreVT.isSimple())
16990b57cec5SDimitry Andric     return false;
17000b57cec5SDimitry Andric 
1701fe6060f1SDimitry Andric   AtomicOrdering Ordering = ST->getSuccessOrdering();
17020b57cec5SDimitry Andric   // In order to lower atomic loads with stronger guarantees we would need to
17030b57cec5SDimitry Andric   // use store.release or insert fences. However these features were only added
17040b57cec5SDimitry Andric   // with PTX ISA 6.0 / sm_70.
17050b57cec5SDimitry Andric   // TODO: Check if we can actually use the new instructions and implement them.
17060b57cec5SDimitry Andric   if (isStrongerThanMonotonic(Ordering))
17070b57cec5SDimitry Andric     return false;
17080b57cec5SDimitry Andric 
17090b57cec5SDimitry Andric   // Address Space Setting
17100b57cec5SDimitry Andric   unsigned int CodeAddrSpace = getCodeAddrSpace(ST);
17110b57cec5SDimitry Andric   unsigned int PointerSize =
17120b57cec5SDimitry Andric       CurDAG->getDataLayout().getPointerSizeInBits(ST->getAddressSpace());
17130b57cec5SDimitry Andric 
17140b57cec5SDimitry Andric   // Volatile Setting
17150b57cec5SDimitry Andric   // - .volatile is only available for .global and .shared
17160b57cec5SDimitry Andric   // - .volatile has the same memory synchronization semantics as .relaxed.sys
17170b57cec5SDimitry Andric   bool isVolatile = ST->isVolatile() || Ordering == AtomicOrdering::Monotonic;
17180b57cec5SDimitry Andric   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
17190b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
17200b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
17210b57cec5SDimitry Andric     isVolatile = false;
17220b57cec5SDimitry Andric 
17230b57cec5SDimitry Andric   // Vector Setting
17240b57cec5SDimitry Andric   MVT SimpleVT = StoreVT.getSimpleVT();
17250b57cec5SDimitry Andric   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
17260b57cec5SDimitry Andric 
17270b57cec5SDimitry Andric   // Type Setting: toType + toTypeWidth
17280b57cec5SDimitry Andric   // - for integer type, always use 'u'
17290b57cec5SDimitry Andric   //
17300b57cec5SDimitry Andric   MVT ScalarVT = SimpleVT.getScalarType();
17310b57cec5SDimitry Andric   unsigned toTypeWidth = ScalarVT.getSizeInBits();
17320b57cec5SDimitry Andric   if (SimpleVT.isVector()) {
17335f757f3fSDimitry Andric     assert((Isv2x16VT(StoreVT) || StoreVT == MVT::v4i8) &&
1734bdd1243dSDimitry Andric            "Unexpected vector type");
17355f757f3fSDimitry Andric     // v2x16 is stored using st.b32
17360b57cec5SDimitry Andric     toTypeWidth = 32;
17370b57cec5SDimitry Andric   }
17380b57cec5SDimitry Andric 
1739bdd1243dSDimitry Andric   unsigned int toType = getLdStRegType(ScalarVT);
17400b57cec5SDimitry Andric 
17410b57cec5SDimitry Andric   // Create the machine instruction DAG
17420b57cec5SDimitry Andric   SDValue Chain = ST->getChain();
17430b57cec5SDimitry Andric   SDValue Value = PlainStore ? PlainStore->getValue() : AtomicStore->getVal();
17440b57cec5SDimitry Andric   SDValue BasePtr = ST->getBasePtr();
17450b57cec5SDimitry Andric   SDValue Addr;
17460b57cec5SDimitry Andric   SDValue Offset, Base;
1747bdd1243dSDimitry Andric   std::optional<unsigned> Opcode;
17480b57cec5SDimitry Andric   MVT::SimpleValueType SourceVT =
17490b57cec5SDimitry Andric       Value.getNode()->getSimpleValueType(0).SimpleTy;
17500b57cec5SDimitry Andric 
17510b57cec5SDimitry Andric   if (SelectDirectAddr(BasePtr, Addr)) {
17520b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_avar, NVPTX::ST_i16_avar,
17530b57cec5SDimitry Andric                              NVPTX::ST_i32_avar, NVPTX::ST_i64_avar,
17540b57cec5SDimitry Andric                              NVPTX::ST_f32_avar, NVPTX::ST_f64_avar);
17550b57cec5SDimitry Andric     if (!Opcode)
17560b57cec5SDimitry Andric       return false;
17570b57cec5SDimitry Andric     SDValue Ops[] = {Value,
17580b57cec5SDimitry Andric                      getI32Imm(isVolatile, dl),
17590b57cec5SDimitry Andric                      getI32Imm(CodeAddrSpace, dl),
17600b57cec5SDimitry Andric                      getI32Imm(vecType, dl),
17610b57cec5SDimitry Andric                      getI32Imm(toType, dl),
17620b57cec5SDimitry Andric                      getI32Imm(toTypeWidth, dl),
17630b57cec5SDimitry Andric                      Addr,
17640b57cec5SDimitry Andric                      Chain};
176581ad6265SDimitry Andric     NVPTXST = CurDAG->getMachineNode(*Opcode, dl, MVT::Other, Ops);
17660b57cec5SDimitry Andric   } else if (PointerSize == 64
17670b57cec5SDimitry Andric                  ? SelectADDRsi64(BasePtr.getNode(), BasePtr, Base, Offset)
17680b57cec5SDimitry Andric                  : SelectADDRsi(BasePtr.getNode(), BasePtr, Base, Offset)) {
17690b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_asi, NVPTX::ST_i16_asi,
17700b57cec5SDimitry Andric                              NVPTX::ST_i32_asi, NVPTX::ST_i64_asi,
17710b57cec5SDimitry Andric                              NVPTX::ST_f32_asi, NVPTX::ST_f64_asi);
17720b57cec5SDimitry Andric     if (!Opcode)
17730b57cec5SDimitry Andric       return false;
17740b57cec5SDimitry Andric     SDValue Ops[] = {Value,
17750b57cec5SDimitry Andric                      getI32Imm(isVolatile, dl),
17760b57cec5SDimitry Andric                      getI32Imm(CodeAddrSpace, dl),
17770b57cec5SDimitry Andric                      getI32Imm(vecType, dl),
17780b57cec5SDimitry Andric                      getI32Imm(toType, dl),
17790b57cec5SDimitry Andric                      getI32Imm(toTypeWidth, dl),
17800b57cec5SDimitry Andric                      Base,
17810b57cec5SDimitry Andric                      Offset,
17820b57cec5SDimitry Andric                      Chain};
178381ad6265SDimitry Andric     NVPTXST = CurDAG->getMachineNode(*Opcode, dl, MVT::Other, Ops);
17840b57cec5SDimitry Andric   } else if (PointerSize == 64
17850b57cec5SDimitry Andric                  ? SelectADDRri64(BasePtr.getNode(), BasePtr, Base, Offset)
17860b57cec5SDimitry Andric                  : SelectADDRri(BasePtr.getNode(), BasePtr, Base, Offset)) {
17870b57cec5SDimitry Andric     if (PointerSize == 64)
178806c3fb27SDimitry Andric       Opcode =
178906c3fb27SDimitry Andric           pickOpcodeForVT(SourceVT, NVPTX::ST_i8_ari_64, NVPTX::ST_i16_ari_64,
179006c3fb27SDimitry Andric                           NVPTX::ST_i32_ari_64, NVPTX::ST_i64_ari_64,
179106c3fb27SDimitry Andric                           NVPTX::ST_f32_ari_64, NVPTX::ST_f64_ari_64);
17920b57cec5SDimitry Andric     else
17930b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_ari, NVPTX::ST_i16_ari,
17940b57cec5SDimitry Andric                                NVPTX::ST_i32_ari, NVPTX::ST_i64_ari,
17950b57cec5SDimitry Andric                                NVPTX::ST_f32_ari, NVPTX::ST_f64_ari);
17960b57cec5SDimitry Andric     if (!Opcode)
17970b57cec5SDimitry Andric       return false;
17980b57cec5SDimitry Andric 
17990b57cec5SDimitry Andric     SDValue Ops[] = {Value,
18000b57cec5SDimitry Andric                      getI32Imm(isVolatile, dl),
18010b57cec5SDimitry Andric                      getI32Imm(CodeAddrSpace, dl),
18020b57cec5SDimitry Andric                      getI32Imm(vecType, dl),
18030b57cec5SDimitry Andric                      getI32Imm(toType, dl),
18040b57cec5SDimitry Andric                      getI32Imm(toTypeWidth, dl),
18050b57cec5SDimitry Andric                      Base,
18060b57cec5SDimitry Andric                      Offset,
18070b57cec5SDimitry Andric                      Chain};
180881ad6265SDimitry Andric     NVPTXST = CurDAG->getMachineNode(*Opcode, dl, MVT::Other, Ops);
18090b57cec5SDimitry Andric   } else {
18100b57cec5SDimitry Andric     if (PointerSize == 64)
18110b57cec5SDimitry Andric       Opcode =
18120b57cec5SDimitry Andric           pickOpcodeForVT(SourceVT, NVPTX::ST_i8_areg_64, NVPTX::ST_i16_areg_64,
18130b57cec5SDimitry Andric                           NVPTX::ST_i32_areg_64, NVPTX::ST_i64_areg_64,
18140b57cec5SDimitry Andric                           NVPTX::ST_f32_areg_64, NVPTX::ST_f64_areg_64);
18150b57cec5SDimitry Andric     else
18160b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_areg, NVPTX::ST_i16_areg,
18170b57cec5SDimitry Andric                                NVPTX::ST_i32_areg, NVPTX::ST_i64_areg,
18180b57cec5SDimitry Andric                                NVPTX::ST_f32_areg, NVPTX::ST_f64_areg);
18190b57cec5SDimitry Andric     if (!Opcode)
18200b57cec5SDimitry Andric       return false;
18210b57cec5SDimitry Andric     SDValue Ops[] = {Value,
18220b57cec5SDimitry Andric                      getI32Imm(isVolatile, dl),
18230b57cec5SDimitry Andric                      getI32Imm(CodeAddrSpace, dl),
18240b57cec5SDimitry Andric                      getI32Imm(vecType, dl),
18250b57cec5SDimitry Andric                      getI32Imm(toType, dl),
18260b57cec5SDimitry Andric                      getI32Imm(toTypeWidth, dl),
18270b57cec5SDimitry Andric                      BasePtr,
18280b57cec5SDimitry Andric                      Chain};
182981ad6265SDimitry Andric     NVPTXST = CurDAG->getMachineNode(*Opcode, dl, MVT::Other, Ops);
18300b57cec5SDimitry Andric   }
18310b57cec5SDimitry Andric 
18320b57cec5SDimitry Andric   if (!NVPTXST)
18330b57cec5SDimitry Andric     return false;
18340b57cec5SDimitry Andric 
18350b57cec5SDimitry Andric   MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
18360b57cec5SDimitry Andric   CurDAG->setNodeMemRefs(cast<MachineSDNode>(NVPTXST), {MemRef});
18370b57cec5SDimitry Andric   ReplaceNode(N, NVPTXST);
18380b57cec5SDimitry Andric   return true;
18390b57cec5SDimitry Andric }
18400b57cec5SDimitry Andric 
18410b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
18420b57cec5SDimitry Andric   SDValue Chain = N->getOperand(0);
18430b57cec5SDimitry Andric   SDValue Op1 = N->getOperand(1);
18440b57cec5SDimitry Andric   SDValue Addr, Offset, Base;
1845bdd1243dSDimitry Andric   std::optional<unsigned> Opcode;
18460b57cec5SDimitry Andric   SDLoc DL(N);
18470b57cec5SDimitry Andric   SDNode *ST;
18480b57cec5SDimitry Andric   EVT EltVT = Op1.getValueType();
18490b57cec5SDimitry Andric   MemSDNode *MemSD = cast<MemSDNode>(N);
18500b57cec5SDimitry Andric   EVT StoreVT = MemSD->getMemoryVT();
18510b57cec5SDimitry Andric 
18520b57cec5SDimitry Andric   // Address Space Setting
18530b57cec5SDimitry Andric   unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
18540b57cec5SDimitry Andric   if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
18550b57cec5SDimitry Andric     report_fatal_error("Cannot store to pointer that points to constant "
18560b57cec5SDimitry Andric                        "memory space");
18570b57cec5SDimitry Andric   }
18580b57cec5SDimitry Andric   unsigned int PointerSize =
18590b57cec5SDimitry Andric       CurDAG->getDataLayout().getPointerSizeInBits(MemSD->getAddressSpace());
18600b57cec5SDimitry Andric 
18610b57cec5SDimitry Andric   // Volatile Setting
18620b57cec5SDimitry Andric   // - .volatile is only availalble for .global and .shared
18630b57cec5SDimitry Andric   bool IsVolatile = MemSD->isVolatile();
18640b57cec5SDimitry Andric   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
18650b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
18660b57cec5SDimitry Andric       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
18670b57cec5SDimitry Andric     IsVolatile = false;
18680b57cec5SDimitry Andric 
18690b57cec5SDimitry Andric   // Type Setting: toType + toTypeWidth
18700b57cec5SDimitry Andric   // - for integer type, always use 'u'
18710b57cec5SDimitry Andric   assert(StoreVT.isSimple() && "Store value is not simple");
18720b57cec5SDimitry Andric   MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
18730b57cec5SDimitry Andric   unsigned ToTypeWidth = ScalarVT.getSizeInBits();
1874bdd1243dSDimitry Andric   unsigned ToType = getLdStRegType(ScalarVT);
18750b57cec5SDimitry Andric 
18760b57cec5SDimitry Andric   SmallVector<SDValue, 12> StOps;
18770b57cec5SDimitry Andric   SDValue N2;
18780b57cec5SDimitry Andric   unsigned VecType;
18790b57cec5SDimitry Andric 
18800b57cec5SDimitry Andric   switch (N->getOpcode()) {
18810b57cec5SDimitry Andric   case NVPTXISD::StoreV2:
18820b57cec5SDimitry Andric     VecType = NVPTX::PTXLdStInstCode::V2;
18830b57cec5SDimitry Andric     StOps.push_back(N->getOperand(1));
18840b57cec5SDimitry Andric     StOps.push_back(N->getOperand(2));
18850b57cec5SDimitry Andric     N2 = N->getOperand(3);
18860b57cec5SDimitry Andric     break;
18870b57cec5SDimitry Andric   case NVPTXISD::StoreV4:
18880b57cec5SDimitry Andric     VecType = NVPTX::PTXLdStInstCode::V4;
18890b57cec5SDimitry Andric     StOps.push_back(N->getOperand(1));
18900b57cec5SDimitry Andric     StOps.push_back(N->getOperand(2));
18910b57cec5SDimitry Andric     StOps.push_back(N->getOperand(3));
18920b57cec5SDimitry Andric     StOps.push_back(N->getOperand(4));
18930b57cec5SDimitry Andric     N2 = N->getOperand(5);
18940b57cec5SDimitry Andric     break;
18950b57cec5SDimitry Andric   default:
18960b57cec5SDimitry Andric     return false;
18970b57cec5SDimitry Andric   }
18980b57cec5SDimitry Andric 
18995f757f3fSDimitry Andric   // v8x16 is a special case. PTX doesn't have st.v8.x16
19005f757f3fSDimitry Andric   // instruction. Instead, we split the vector into v2x16 chunks and
19010b57cec5SDimitry Andric   // store them with st.v4.b32.
19025f757f3fSDimitry Andric   if (Isv2x16VT(EltVT)) {
19030b57cec5SDimitry Andric     assert(N->getOpcode() == NVPTXISD::StoreV4 && "Unexpected load opcode.");
19040b57cec5SDimitry Andric     EltVT = MVT::i32;
19050b57cec5SDimitry Andric     ToType = NVPTX::PTXLdStInstCode::Untyped;
19060b57cec5SDimitry Andric     ToTypeWidth = 32;
19070b57cec5SDimitry Andric   }
19080b57cec5SDimitry Andric 
19090b57cec5SDimitry Andric   StOps.push_back(getI32Imm(IsVolatile, DL));
19100b57cec5SDimitry Andric   StOps.push_back(getI32Imm(CodeAddrSpace, DL));
19110b57cec5SDimitry Andric   StOps.push_back(getI32Imm(VecType, DL));
19120b57cec5SDimitry Andric   StOps.push_back(getI32Imm(ToType, DL));
19130b57cec5SDimitry Andric   StOps.push_back(getI32Imm(ToTypeWidth, DL));
19140b57cec5SDimitry Andric 
19150b57cec5SDimitry Andric   if (SelectDirectAddr(N2, Addr)) {
19160b57cec5SDimitry Andric     switch (N->getOpcode()) {
19170b57cec5SDimitry Andric     default:
19180b57cec5SDimitry Andric       return false;
19190b57cec5SDimitry Andric     case NVPTXISD::StoreV2:
19200b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
19210b57cec5SDimitry Andric                                NVPTX::STV_i8_v2_avar, NVPTX::STV_i16_v2_avar,
19220b57cec5SDimitry Andric                                NVPTX::STV_i32_v2_avar, NVPTX::STV_i64_v2_avar,
19230b57cec5SDimitry Andric                                NVPTX::STV_f32_v2_avar, NVPTX::STV_f64_v2_avar);
19240b57cec5SDimitry Andric       break;
19250b57cec5SDimitry Andric     case NVPTXISD::StoreV4:
1926bdd1243dSDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
1927bdd1243dSDimitry Andric                                NVPTX::STV_i8_v4_avar, NVPTX::STV_i16_v4_avar,
1928bdd1243dSDimitry Andric                                NVPTX::STV_i32_v4_avar, std::nullopt,
1929bdd1243dSDimitry Andric                                NVPTX::STV_f32_v4_avar, std::nullopt);
19300b57cec5SDimitry Andric       break;
19310b57cec5SDimitry Andric     }
19320b57cec5SDimitry Andric     StOps.push_back(Addr);
19330b57cec5SDimitry Andric   } else if (PointerSize == 64 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
19340b57cec5SDimitry Andric                                : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
19350b57cec5SDimitry Andric     switch (N->getOpcode()) {
19360b57cec5SDimitry Andric     default:
19370b57cec5SDimitry Andric       return false;
19380b57cec5SDimitry Andric     case NVPTXISD::StoreV2:
19390b57cec5SDimitry Andric       Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
19400b57cec5SDimitry Andric                                NVPTX::STV_i8_v2_asi, NVPTX::STV_i16_v2_asi,
19410b57cec5SDimitry Andric                                NVPTX::STV_i32_v2_asi, NVPTX::STV_i64_v2_asi,
19420b57cec5SDimitry Andric                                NVPTX::STV_f32_v2_asi, NVPTX::STV_f64_v2_asi);
19430b57cec5SDimitry Andric       break;
19440b57cec5SDimitry Andric     case NVPTXISD::StoreV4:
194506c3fb27SDimitry Andric       Opcode =
194606c3fb27SDimitry Andric           pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_asi,
194706c3fb27SDimitry Andric                           NVPTX::STV_i16_v4_asi, NVPTX::STV_i32_v4_asi,
194806c3fb27SDimitry Andric                           std::nullopt, NVPTX::STV_f32_v4_asi, std::nullopt);
19490b57cec5SDimitry Andric       break;
19500b57cec5SDimitry Andric     }
19510b57cec5SDimitry Andric     StOps.push_back(Base);
19520b57cec5SDimitry Andric     StOps.push_back(Offset);
19530b57cec5SDimitry Andric   } else if (PointerSize == 64 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
19540b57cec5SDimitry Andric                                : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
19550b57cec5SDimitry Andric     if (PointerSize == 64) {
19560b57cec5SDimitry Andric       switch (N->getOpcode()) {
19570b57cec5SDimitry Andric       default:
19580b57cec5SDimitry Andric         return false;
19590b57cec5SDimitry Andric       case NVPTXISD::StoreV2:
196006c3fb27SDimitry Andric         Opcode =
196106c3fb27SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
196206c3fb27SDimitry Andric                             NVPTX::STV_i8_v2_ari_64, NVPTX::STV_i16_v2_ari_64,
196306c3fb27SDimitry Andric                             NVPTX::STV_i32_v2_ari_64, NVPTX::STV_i64_v2_ari_64,
196406c3fb27SDimitry Andric                             NVPTX::STV_f32_v2_ari_64, NVPTX::STV_f64_v2_ari_64);
19650b57cec5SDimitry Andric         break;
19660b57cec5SDimitry Andric       case NVPTXISD::StoreV4:
19670b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(
19680b57cec5SDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_ari_64,
1969bdd1243dSDimitry Andric             NVPTX::STV_i16_v4_ari_64, NVPTX::STV_i32_v4_ari_64, std::nullopt,
1970bdd1243dSDimitry Andric             NVPTX::STV_f32_v4_ari_64, std::nullopt);
19710b57cec5SDimitry Andric         break;
19720b57cec5SDimitry Andric       }
19730b57cec5SDimitry Andric     } else {
19740b57cec5SDimitry Andric       switch (N->getOpcode()) {
19750b57cec5SDimitry Andric       default:
19760b57cec5SDimitry Andric         return false;
19770b57cec5SDimitry Andric       case NVPTXISD::StoreV2:
19780b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
19790b57cec5SDimitry Andric                                  NVPTX::STV_i8_v2_ari, NVPTX::STV_i16_v2_ari,
19800b57cec5SDimitry Andric                                  NVPTX::STV_i32_v2_ari, NVPTX::STV_i64_v2_ari,
19810b57cec5SDimitry Andric                                  NVPTX::STV_f32_v2_ari, NVPTX::STV_f64_v2_ari);
19820b57cec5SDimitry Andric         break;
19830b57cec5SDimitry Andric       case NVPTXISD::StoreV4:
1984bdd1243dSDimitry Andric         Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
1985bdd1243dSDimitry Andric                                  NVPTX::STV_i8_v4_ari, NVPTX::STV_i16_v4_ari,
1986bdd1243dSDimitry Andric                                  NVPTX::STV_i32_v4_ari, std::nullopt,
1987bdd1243dSDimitry Andric                                  NVPTX::STV_f32_v4_ari, std::nullopt);
19880b57cec5SDimitry Andric         break;
19890b57cec5SDimitry Andric       }
19900b57cec5SDimitry Andric     }
19910b57cec5SDimitry Andric     StOps.push_back(Base);
19920b57cec5SDimitry Andric     StOps.push_back(Offset);
19930b57cec5SDimitry Andric   } else {
19940b57cec5SDimitry Andric     if (PointerSize == 64) {
19950b57cec5SDimitry Andric       switch (N->getOpcode()) {
19960b57cec5SDimitry Andric       default:
19970b57cec5SDimitry Andric         return false;
19980b57cec5SDimitry Andric       case NVPTXISD::StoreV2:
19990b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(
20000b57cec5SDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v2_areg_64,
20010b57cec5SDimitry Andric             NVPTX::STV_i16_v2_areg_64, NVPTX::STV_i32_v2_areg_64,
200206c3fb27SDimitry Andric             NVPTX::STV_i64_v2_areg_64, NVPTX::STV_f32_v2_areg_64,
20030b57cec5SDimitry Andric             NVPTX::STV_f64_v2_areg_64);
20040b57cec5SDimitry Andric         break;
20050b57cec5SDimitry Andric       case NVPTXISD::StoreV4:
20060b57cec5SDimitry Andric         Opcode = pickOpcodeForVT(
20070b57cec5SDimitry Andric             EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_areg_64,
2008bdd1243dSDimitry Andric             NVPTX::STV_i16_v4_areg_64, NVPTX::STV_i32_v4_areg_64, std::nullopt,
2009bdd1243dSDimitry Andric             NVPTX::STV_f32_v4_areg_64, std::nullopt);
20100b57cec5SDimitry Andric         break;
20110b57cec5SDimitry Andric       }
20120b57cec5SDimitry Andric     } else {
20130b57cec5SDimitry Andric       switch (N->getOpcode()) {
20140b57cec5SDimitry Andric       default:
20150b57cec5SDimitry Andric         return false;
20160b57cec5SDimitry Andric       case NVPTXISD::StoreV2:
20170b57cec5SDimitry Andric         Opcode =
20180b57cec5SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v2_areg,
20190b57cec5SDimitry Andric                             NVPTX::STV_i16_v2_areg, NVPTX::STV_i32_v2_areg,
202006c3fb27SDimitry Andric                             NVPTX::STV_i64_v2_areg, NVPTX::STV_f32_v2_areg,
20210b57cec5SDimitry Andric                             NVPTX::STV_f64_v2_areg);
20220b57cec5SDimitry Andric         break;
20230b57cec5SDimitry Andric       case NVPTXISD::StoreV4:
202406c3fb27SDimitry Andric         Opcode =
202506c3fb27SDimitry Andric             pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_areg,
202606c3fb27SDimitry Andric                             NVPTX::STV_i16_v4_areg, NVPTX::STV_i32_v4_areg,
202706c3fb27SDimitry Andric                             std::nullopt, NVPTX::STV_f32_v4_areg, std::nullopt);
20280b57cec5SDimitry Andric         break;
20290b57cec5SDimitry Andric       }
20300b57cec5SDimitry Andric     }
20310b57cec5SDimitry Andric     StOps.push_back(N2);
20320b57cec5SDimitry Andric   }
20330b57cec5SDimitry Andric 
20340b57cec5SDimitry Andric   if (!Opcode)
20350b57cec5SDimitry Andric     return false;
20360b57cec5SDimitry Andric 
20370b57cec5SDimitry Andric   StOps.push_back(Chain);
20380b57cec5SDimitry Andric 
203981ad6265SDimitry Andric   ST = CurDAG->getMachineNode(*Opcode, DL, MVT::Other, StOps);
20400b57cec5SDimitry Andric 
20410b57cec5SDimitry Andric   MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
20420b57cec5SDimitry Andric   CurDAG->setNodeMemRefs(cast<MachineSDNode>(ST), {MemRef});
20430b57cec5SDimitry Andric 
20440b57cec5SDimitry Andric   ReplaceNode(N, ST);
20450b57cec5SDimitry Andric   return true;
20460b57cec5SDimitry Andric }
20470b57cec5SDimitry Andric 
20480b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryLoadParam(SDNode *Node) {
20490b57cec5SDimitry Andric   SDValue Chain = Node->getOperand(0);
20500b57cec5SDimitry Andric   SDValue Offset = Node->getOperand(2);
205106c3fb27SDimitry Andric   SDValue Glue = Node->getOperand(3);
20520b57cec5SDimitry Andric   SDLoc DL(Node);
20530b57cec5SDimitry Andric   MemSDNode *Mem = cast<MemSDNode>(Node);
20540b57cec5SDimitry Andric 
20550b57cec5SDimitry Andric   unsigned VecSize;
20560b57cec5SDimitry Andric   switch (Node->getOpcode()) {
20570b57cec5SDimitry Andric   default:
20580b57cec5SDimitry Andric     return false;
20590b57cec5SDimitry Andric   case NVPTXISD::LoadParam:
20600b57cec5SDimitry Andric     VecSize = 1;
20610b57cec5SDimitry Andric     break;
20620b57cec5SDimitry Andric   case NVPTXISD::LoadParamV2:
20630b57cec5SDimitry Andric     VecSize = 2;
20640b57cec5SDimitry Andric     break;
20650b57cec5SDimitry Andric   case NVPTXISD::LoadParamV4:
20660b57cec5SDimitry Andric     VecSize = 4;
20670b57cec5SDimitry Andric     break;
20680b57cec5SDimitry Andric   }
20690b57cec5SDimitry Andric 
20700b57cec5SDimitry Andric   EVT EltVT = Node->getValueType(0);
20710b57cec5SDimitry Andric   EVT MemVT = Mem->getMemoryVT();
20720b57cec5SDimitry Andric 
2073bdd1243dSDimitry Andric   std::optional<unsigned> Opcode;
20740b57cec5SDimitry Andric 
20750b57cec5SDimitry Andric   switch (VecSize) {
20760b57cec5SDimitry Andric   default:
20770b57cec5SDimitry Andric     return false;
20780b57cec5SDimitry Andric   case 1:
20790b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(MemVT.getSimpleVT().SimpleTy,
20800b57cec5SDimitry Andric                              NVPTX::LoadParamMemI8, NVPTX::LoadParamMemI16,
20810b57cec5SDimitry Andric                              NVPTX::LoadParamMemI32, NVPTX::LoadParamMemI64,
20820b57cec5SDimitry Andric                              NVPTX::LoadParamMemF32, NVPTX::LoadParamMemF64);
20830b57cec5SDimitry Andric     break;
20840b57cec5SDimitry Andric   case 2:
20850b57cec5SDimitry Andric     Opcode =
20860b57cec5SDimitry Andric         pickOpcodeForVT(MemVT.getSimpleVT().SimpleTy, NVPTX::LoadParamMemV2I8,
20870b57cec5SDimitry Andric                         NVPTX::LoadParamMemV2I16, NVPTX::LoadParamMemV2I32,
208806c3fb27SDimitry Andric                         NVPTX::LoadParamMemV2I64, NVPTX::LoadParamMemV2F32,
20890b57cec5SDimitry Andric                         NVPTX::LoadParamMemV2F64);
20900b57cec5SDimitry Andric     break;
20910b57cec5SDimitry Andric   case 4:
209206c3fb27SDimitry Andric     Opcode =
209306c3fb27SDimitry Andric         pickOpcodeForVT(MemVT.getSimpleVT().SimpleTy, NVPTX::LoadParamMemV4I8,
209406c3fb27SDimitry Andric                         NVPTX::LoadParamMemV4I16, NVPTX::LoadParamMemV4I32,
209506c3fb27SDimitry Andric                         std::nullopt, NVPTX::LoadParamMemV4F32, std::nullopt);
20960b57cec5SDimitry Andric     break;
20970b57cec5SDimitry Andric   }
20980b57cec5SDimitry Andric   if (!Opcode)
20990b57cec5SDimitry Andric     return false;
21000b57cec5SDimitry Andric 
21010b57cec5SDimitry Andric   SDVTList VTs;
21020b57cec5SDimitry Andric   if (VecSize == 1) {
21030b57cec5SDimitry Andric     VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
21040b57cec5SDimitry Andric   } else if (VecSize == 2) {
21050b57cec5SDimitry Andric     VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
21060b57cec5SDimitry Andric   } else {
21070b57cec5SDimitry Andric     EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
21080b57cec5SDimitry Andric     VTs = CurDAG->getVTList(EVTs);
21090b57cec5SDimitry Andric   }
21100b57cec5SDimitry Andric 
21111db9f3b2SDimitry Andric   unsigned OffsetVal = Offset->getAsZExtVal();
21120b57cec5SDimitry Andric 
21130b57cec5SDimitry Andric   SmallVector<SDValue, 2> Ops;
21140b57cec5SDimitry Andric   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
21150b57cec5SDimitry Andric   Ops.push_back(Chain);
211606c3fb27SDimitry Andric   Ops.push_back(Glue);
21170b57cec5SDimitry Andric 
211881ad6265SDimitry Andric   ReplaceNode(Node, CurDAG->getMachineNode(*Opcode, DL, VTs, Ops));
21190b57cec5SDimitry Andric   return true;
21200b57cec5SDimitry Andric }
21210b57cec5SDimitry Andric 
21220b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) {
21230b57cec5SDimitry Andric   SDLoc DL(N);
21240b57cec5SDimitry Andric   SDValue Chain = N->getOperand(0);
21250b57cec5SDimitry Andric   SDValue Offset = N->getOperand(1);
21261db9f3b2SDimitry Andric   unsigned OffsetVal = Offset->getAsZExtVal();
21270b57cec5SDimitry Andric   MemSDNode *Mem = cast<MemSDNode>(N);
21280b57cec5SDimitry Andric 
21290b57cec5SDimitry Andric   // How many elements do we have?
21300b57cec5SDimitry Andric   unsigned NumElts = 1;
21310b57cec5SDimitry Andric   switch (N->getOpcode()) {
21320b57cec5SDimitry Andric   default:
21330b57cec5SDimitry Andric     return false;
21340b57cec5SDimitry Andric   case NVPTXISD::StoreRetval:
21350b57cec5SDimitry Andric     NumElts = 1;
21360b57cec5SDimitry Andric     break;
21370b57cec5SDimitry Andric   case NVPTXISD::StoreRetvalV2:
21380b57cec5SDimitry Andric     NumElts = 2;
21390b57cec5SDimitry Andric     break;
21400b57cec5SDimitry Andric   case NVPTXISD::StoreRetvalV4:
21410b57cec5SDimitry Andric     NumElts = 4;
21420b57cec5SDimitry Andric     break;
21430b57cec5SDimitry Andric   }
21440b57cec5SDimitry Andric 
21450b57cec5SDimitry Andric   // Build vector of operands
21460b57cec5SDimitry Andric   SmallVector<SDValue, 6> Ops;
21470b57cec5SDimitry Andric   for (unsigned i = 0; i < NumElts; ++i)
21480b57cec5SDimitry Andric     Ops.push_back(N->getOperand(i + 2));
21490b57cec5SDimitry Andric   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
21500b57cec5SDimitry Andric   Ops.push_back(Chain);
21510b57cec5SDimitry Andric 
21520b57cec5SDimitry Andric   // Determine target opcode
21530b57cec5SDimitry Andric   // If we have an i1, use an 8-bit store. The lowering code in
21540b57cec5SDimitry Andric   // NVPTXISelLowering will have already emitted an upcast.
2155bdd1243dSDimitry Andric   std::optional<unsigned> Opcode = 0;
21560b57cec5SDimitry Andric   switch (NumElts) {
21570b57cec5SDimitry Andric   default:
21580b57cec5SDimitry Andric     return false;
21590b57cec5SDimitry Andric   case 1:
21600b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
21610b57cec5SDimitry Andric                              NVPTX::StoreRetvalI8, NVPTX::StoreRetvalI16,
21620b57cec5SDimitry Andric                              NVPTX::StoreRetvalI32, NVPTX::StoreRetvalI64,
21630b57cec5SDimitry Andric                              NVPTX::StoreRetvalF32, NVPTX::StoreRetvalF64);
2164*0fca6ea1SDimitry Andric     if (Opcode == NVPTX::StoreRetvalI8) {
2165*0fca6ea1SDimitry Andric       // Fine tune the opcode depending on the size of the operand.
2166*0fca6ea1SDimitry Andric       // This helps to avoid creating redundant COPY instructions in
2167*0fca6ea1SDimitry Andric       // InstrEmitter::AddRegisterOperand().
2168*0fca6ea1SDimitry Andric       switch (Ops[0].getSimpleValueType().SimpleTy) {
2169*0fca6ea1SDimitry Andric       default:
2170*0fca6ea1SDimitry Andric         break;
2171*0fca6ea1SDimitry Andric       case MVT::i32:
2172*0fca6ea1SDimitry Andric         Opcode = NVPTX::StoreRetvalI8TruncI32;
2173*0fca6ea1SDimitry Andric         break;
2174*0fca6ea1SDimitry Andric       case MVT::i64:
2175*0fca6ea1SDimitry Andric         Opcode = NVPTX::StoreRetvalI8TruncI64;
2176*0fca6ea1SDimitry Andric         break;
2177*0fca6ea1SDimitry Andric       }
2178*0fca6ea1SDimitry Andric     }
21790b57cec5SDimitry Andric     break;
21800b57cec5SDimitry Andric   case 2:
21810b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
21820b57cec5SDimitry Andric                              NVPTX::StoreRetvalV2I8, NVPTX::StoreRetvalV2I16,
21830b57cec5SDimitry Andric                              NVPTX::StoreRetvalV2I32, NVPTX::StoreRetvalV2I64,
21840b57cec5SDimitry Andric                              NVPTX::StoreRetvalV2F32, NVPTX::StoreRetvalV2F64);
21850b57cec5SDimitry Andric     break;
21860b57cec5SDimitry Andric   case 4:
21870b57cec5SDimitry Andric     Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
21880b57cec5SDimitry Andric                              NVPTX::StoreRetvalV4I8, NVPTX::StoreRetvalV4I16,
2189bdd1243dSDimitry Andric                              NVPTX::StoreRetvalV4I32, std::nullopt,
2190bdd1243dSDimitry Andric                              NVPTX::StoreRetvalV4F32, std::nullopt);
21910b57cec5SDimitry Andric     break;
21920b57cec5SDimitry Andric   }
21930b57cec5SDimitry Andric   if (!Opcode)
21940b57cec5SDimitry Andric     return false;
21950b57cec5SDimitry Andric 
219681ad6265SDimitry Andric   SDNode *Ret = CurDAG->getMachineNode(*Opcode, DL, MVT::Other, Ops);
21970b57cec5SDimitry Andric   MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
21980b57cec5SDimitry Andric   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ret), {MemRef});
21990b57cec5SDimitry Andric 
22000b57cec5SDimitry Andric   ReplaceNode(N, Ret);
22010b57cec5SDimitry Andric   return true;
22020b57cec5SDimitry Andric }
22030b57cec5SDimitry Andric 
2204*0fca6ea1SDimitry Andric // Helpers for constructing opcode (ex: NVPTX::StoreParamV4F32_iiri)
2205*0fca6ea1SDimitry Andric #define getOpcV2H(ty, opKind0, opKind1)                                        \
2206*0fca6ea1SDimitry Andric   NVPTX::StoreParamV2##ty##_##opKind0##opKind1
2207*0fca6ea1SDimitry Andric 
2208*0fca6ea1SDimitry Andric #define getOpcV2H1(ty, opKind0, isImm1)                                        \
2209*0fca6ea1SDimitry Andric   (isImm1) ? getOpcV2H(ty, opKind0, i) : getOpcV2H(ty, opKind0, r)
2210*0fca6ea1SDimitry Andric 
2211*0fca6ea1SDimitry Andric #define getOpcodeForVectorStParamV2(ty, isimm)                                 \
2212*0fca6ea1SDimitry Andric   (isimm[0]) ? getOpcV2H1(ty, i, isimm[1]) : getOpcV2H1(ty, r, isimm[1])
2213*0fca6ea1SDimitry Andric 
2214*0fca6ea1SDimitry Andric #define getOpcV4H(ty, opKind0, opKind1, opKind2, opKind3)                      \
2215*0fca6ea1SDimitry Andric   NVPTX::StoreParamV4##ty##_##opKind0##opKind1##opKind2##opKind3
2216*0fca6ea1SDimitry Andric 
2217*0fca6ea1SDimitry Andric #define getOpcV4H3(ty, opKind0, opKind1, opKind2, isImm3)                      \
2218*0fca6ea1SDimitry Andric   (isImm3) ? getOpcV4H(ty, opKind0, opKind1, opKind2, i)                       \
2219*0fca6ea1SDimitry Andric            : getOpcV4H(ty, opKind0, opKind1, opKind2, r)
2220*0fca6ea1SDimitry Andric 
2221*0fca6ea1SDimitry Andric #define getOpcV4H2(ty, opKind0, opKind1, isImm2, isImm3)                       \
2222*0fca6ea1SDimitry Andric   (isImm2) ? getOpcV4H3(ty, opKind0, opKind1, i, isImm3)                       \
2223*0fca6ea1SDimitry Andric            : getOpcV4H3(ty, opKind0, opKind1, r, isImm3)
2224*0fca6ea1SDimitry Andric 
2225*0fca6ea1SDimitry Andric #define getOpcV4H1(ty, opKind0, isImm1, isImm2, isImm3)                        \
2226*0fca6ea1SDimitry Andric   (isImm1) ? getOpcV4H2(ty, opKind0, i, isImm2, isImm3)                        \
2227*0fca6ea1SDimitry Andric            : getOpcV4H2(ty, opKind0, r, isImm2, isImm3)
2228*0fca6ea1SDimitry Andric 
2229*0fca6ea1SDimitry Andric #define getOpcodeForVectorStParamV4(ty, isimm)                                 \
2230*0fca6ea1SDimitry Andric   (isimm[0]) ? getOpcV4H1(ty, i, isimm[1], isimm[2], isimm[3])                 \
2231*0fca6ea1SDimitry Andric              : getOpcV4H1(ty, r, isimm[1], isimm[2], isimm[3])
2232*0fca6ea1SDimitry Andric 
2233*0fca6ea1SDimitry Andric #define getOpcodeForVectorStParam(n, ty, isimm)                                \
2234*0fca6ea1SDimitry Andric   (n == 2) ? getOpcodeForVectorStParamV2(ty, isimm)                            \
2235*0fca6ea1SDimitry Andric            : getOpcodeForVectorStParamV4(ty, isimm)
2236*0fca6ea1SDimitry Andric 
2237*0fca6ea1SDimitry Andric static unsigned pickOpcodeForVectorStParam(SmallVector<SDValue, 8> &Ops,
2238*0fca6ea1SDimitry Andric                                            unsigned NumElts,
2239*0fca6ea1SDimitry Andric                                            MVT::SimpleValueType MemTy,
2240*0fca6ea1SDimitry Andric                                            SelectionDAG *CurDAG, SDLoc DL) {
2241*0fca6ea1SDimitry Andric   // Determine which inputs are registers and immediates make new operators
2242*0fca6ea1SDimitry Andric   // with constant values
2243*0fca6ea1SDimitry Andric   SmallVector<bool, 4> IsImm(NumElts, false);
2244*0fca6ea1SDimitry Andric   for (unsigned i = 0; i < NumElts; i++) {
2245*0fca6ea1SDimitry Andric     IsImm[i] = (isa<ConstantSDNode>(Ops[i]) || isa<ConstantFPSDNode>(Ops[i]));
2246*0fca6ea1SDimitry Andric     if (IsImm[i]) {
2247*0fca6ea1SDimitry Andric       SDValue Imm = Ops[i];
2248*0fca6ea1SDimitry Andric       if (MemTy == MVT::f32 || MemTy == MVT::f64) {
2249*0fca6ea1SDimitry Andric         const ConstantFPSDNode *ConstImm = cast<ConstantFPSDNode>(Imm);
2250*0fca6ea1SDimitry Andric         const ConstantFP *CF = ConstImm->getConstantFPValue();
2251*0fca6ea1SDimitry Andric         Imm = CurDAG->getTargetConstantFP(*CF, DL, Imm->getValueType(0));
2252*0fca6ea1SDimitry Andric       } else {
2253*0fca6ea1SDimitry Andric         const ConstantSDNode *ConstImm = cast<ConstantSDNode>(Imm);
2254*0fca6ea1SDimitry Andric         const ConstantInt *CI = ConstImm->getConstantIntValue();
2255*0fca6ea1SDimitry Andric         Imm = CurDAG->getTargetConstant(*CI, DL, Imm->getValueType(0));
2256*0fca6ea1SDimitry Andric       }
2257*0fca6ea1SDimitry Andric       Ops[i] = Imm;
2258*0fca6ea1SDimitry Andric     }
2259*0fca6ea1SDimitry Andric   }
2260*0fca6ea1SDimitry Andric 
2261*0fca6ea1SDimitry Andric   // Get opcode for MemTy, size, and register/immediate operand ordering
2262*0fca6ea1SDimitry Andric   switch (MemTy) {
2263*0fca6ea1SDimitry Andric   case MVT::i8:
2264*0fca6ea1SDimitry Andric     return getOpcodeForVectorStParam(NumElts, I8, IsImm);
2265*0fca6ea1SDimitry Andric   case MVT::i16:
2266*0fca6ea1SDimitry Andric     return getOpcodeForVectorStParam(NumElts, I16, IsImm);
2267*0fca6ea1SDimitry Andric   case MVT::i32:
2268*0fca6ea1SDimitry Andric     return getOpcodeForVectorStParam(NumElts, I32, IsImm);
2269*0fca6ea1SDimitry Andric   case MVT::i64:
2270*0fca6ea1SDimitry Andric     assert(NumElts == 2 && "MVT too large for NumElts > 2");
2271*0fca6ea1SDimitry Andric     return getOpcodeForVectorStParamV2(I64, IsImm);
2272*0fca6ea1SDimitry Andric   case MVT::f32:
2273*0fca6ea1SDimitry Andric     return getOpcodeForVectorStParam(NumElts, F32, IsImm);
2274*0fca6ea1SDimitry Andric   case MVT::f64:
2275*0fca6ea1SDimitry Andric     assert(NumElts == 2 && "MVT too large for NumElts > 2");
2276*0fca6ea1SDimitry Andric     return getOpcodeForVectorStParamV2(F64, IsImm);
2277*0fca6ea1SDimitry Andric 
2278*0fca6ea1SDimitry Andric   // These cases don't support immediates, just use the all register version
2279*0fca6ea1SDimitry Andric   // and generate moves.
2280*0fca6ea1SDimitry Andric   case MVT::i1:
2281*0fca6ea1SDimitry Andric     return (NumElts == 2) ? NVPTX::StoreParamV2I8_rr
2282*0fca6ea1SDimitry Andric                           : NVPTX::StoreParamV4I8_rrrr;
2283*0fca6ea1SDimitry Andric   case MVT::f16:
2284*0fca6ea1SDimitry Andric   case MVT::bf16:
2285*0fca6ea1SDimitry Andric     return (NumElts == 2) ? NVPTX::StoreParamV2I16_rr
2286*0fca6ea1SDimitry Andric                           : NVPTX::StoreParamV4I16_rrrr;
2287*0fca6ea1SDimitry Andric   case MVT::v2f16:
2288*0fca6ea1SDimitry Andric   case MVT::v2bf16:
2289*0fca6ea1SDimitry Andric   case MVT::v2i16:
2290*0fca6ea1SDimitry Andric   case MVT::v4i8:
2291*0fca6ea1SDimitry Andric     return (NumElts == 2) ? NVPTX::StoreParamV2I32_rr
2292*0fca6ea1SDimitry Andric                           : NVPTX::StoreParamV4I32_rrrr;
2293*0fca6ea1SDimitry Andric   default:
2294*0fca6ea1SDimitry Andric     llvm_unreachable("Cannot select st.param for unknown MemTy");
2295*0fca6ea1SDimitry Andric   }
2296*0fca6ea1SDimitry Andric }
2297*0fca6ea1SDimitry Andric 
22980b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
22990b57cec5SDimitry Andric   SDLoc DL(N);
23000b57cec5SDimitry Andric   SDValue Chain = N->getOperand(0);
23010b57cec5SDimitry Andric   SDValue Param = N->getOperand(1);
23021db9f3b2SDimitry Andric   unsigned ParamVal = Param->getAsZExtVal();
23030b57cec5SDimitry Andric   SDValue Offset = N->getOperand(2);
23041db9f3b2SDimitry Andric   unsigned OffsetVal = Offset->getAsZExtVal();
23050b57cec5SDimitry Andric   MemSDNode *Mem = cast<MemSDNode>(N);
230606c3fb27SDimitry Andric   SDValue Glue = N->getOperand(N->getNumOperands() - 1);
23070b57cec5SDimitry Andric 
23080b57cec5SDimitry Andric   // How many elements do we have?
2309*0fca6ea1SDimitry Andric   unsigned NumElts;
23100b57cec5SDimitry Andric   switch (N->getOpcode()) {
23110b57cec5SDimitry Andric   default:
2312*0fca6ea1SDimitry Andric     llvm_unreachable("Unexpected opcode");
23130b57cec5SDimitry Andric   case NVPTXISD::StoreParamU32:
23140b57cec5SDimitry Andric   case NVPTXISD::StoreParamS32:
23150b57cec5SDimitry Andric   case NVPTXISD::StoreParam:
23160b57cec5SDimitry Andric     NumElts = 1;
23170b57cec5SDimitry Andric     break;
23180b57cec5SDimitry Andric   case NVPTXISD::StoreParamV2:
23190b57cec5SDimitry Andric     NumElts = 2;
23200b57cec5SDimitry Andric     break;
23210b57cec5SDimitry Andric   case NVPTXISD::StoreParamV4:
23220b57cec5SDimitry Andric     NumElts = 4;
23230b57cec5SDimitry Andric     break;
23240b57cec5SDimitry Andric   }
23250b57cec5SDimitry Andric 
23260b57cec5SDimitry Andric   // Build vector of operands
23270b57cec5SDimitry Andric   SmallVector<SDValue, 8> Ops;
23280b57cec5SDimitry Andric   for (unsigned i = 0; i < NumElts; ++i)
23290b57cec5SDimitry Andric     Ops.push_back(N->getOperand(i + 3));
23300b57cec5SDimitry Andric   Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32));
23310b57cec5SDimitry Andric   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
23320b57cec5SDimitry Andric   Ops.push_back(Chain);
233306c3fb27SDimitry Andric   Ops.push_back(Glue);
23340b57cec5SDimitry Andric 
23350b57cec5SDimitry Andric   // Determine target opcode
23360b57cec5SDimitry Andric   // If we have an i1, use an 8-bit store. The lowering code in
23370b57cec5SDimitry Andric   // NVPTXISelLowering will have already emitted an upcast.
2338*0fca6ea1SDimitry Andric   std::optional<unsigned> Opcode;
23390b57cec5SDimitry Andric   switch (N->getOpcode()) {
23400b57cec5SDimitry Andric   default:
23410b57cec5SDimitry Andric     switch (NumElts) {
23420b57cec5SDimitry Andric     default:
2343*0fca6ea1SDimitry Andric       llvm_unreachable("Unexpected NumElts");
2344*0fca6ea1SDimitry Andric     case 1: {
2345*0fca6ea1SDimitry Andric       MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
2346*0fca6ea1SDimitry Andric       SDValue Imm = Ops[0];
2347*0fca6ea1SDimitry Andric       if (MemTy != MVT::f16 && MemTy != MVT::v2f16 &&
2348*0fca6ea1SDimitry Andric           (isa<ConstantSDNode>(Imm) || isa<ConstantFPSDNode>(Imm))) {
2349*0fca6ea1SDimitry Andric         // Convert immediate to target constant
2350*0fca6ea1SDimitry Andric         if (MemTy == MVT::f32 || MemTy == MVT::f64) {
2351*0fca6ea1SDimitry Andric           const ConstantFPSDNode *ConstImm = cast<ConstantFPSDNode>(Imm);
2352*0fca6ea1SDimitry Andric           const ConstantFP *CF = ConstImm->getConstantFPValue();
2353*0fca6ea1SDimitry Andric           Imm = CurDAG->getTargetConstantFP(*CF, DL, Imm->getValueType(0));
2354*0fca6ea1SDimitry Andric         } else {
2355*0fca6ea1SDimitry Andric           const ConstantSDNode *ConstImm = cast<ConstantSDNode>(Imm);
2356*0fca6ea1SDimitry Andric           const ConstantInt *CI = ConstImm->getConstantIntValue();
2357*0fca6ea1SDimitry Andric           Imm = CurDAG->getTargetConstant(*CI, DL, Imm->getValueType(0));
2358*0fca6ea1SDimitry Andric         }
2359*0fca6ea1SDimitry Andric         Ops[0] = Imm;
2360*0fca6ea1SDimitry Andric         // Use immediate version of store param
2361*0fca6ea1SDimitry Andric         Opcode = pickOpcodeForVT(MemTy, NVPTX::StoreParamI8_i,
2362*0fca6ea1SDimitry Andric                                  NVPTX::StoreParamI16_i, NVPTX::StoreParamI32_i,
2363*0fca6ea1SDimitry Andric                                  NVPTX::StoreParamI64_i, NVPTX::StoreParamF32_i,
2364*0fca6ea1SDimitry Andric                                  NVPTX::StoreParamF64_i);
2365*0fca6ea1SDimitry Andric       } else
2366*0fca6ea1SDimitry Andric         Opcode =
2367*0fca6ea1SDimitry Andric             pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
2368*0fca6ea1SDimitry Andric                             NVPTX::StoreParamI8_r, NVPTX::StoreParamI16_r,
2369*0fca6ea1SDimitry Andric                             NVPTX::StoreParamI32_r, NVPTX::StoreParamI64_r,
2370*0fca6ea1SDimitry Andric                             NVPTX::StoreParamF32_r, NVPTX::StoreParamF64_r);
2371*0fca6ea1SDimitry Andric       if (Opcode == NVPTX::StoreParamI8_r) {
2372*0fca6ea1SDimitry Andric         // Fine tune the opcode depending on the size of the operand.
2373*0fca6ea1SDimitry Andric         // This helps to avoid creating redundant COPY instructions in
2374*0fca6ea1SDimitry Andric         // InstrEmitter::AddRegisterOperand().
2375*0fca6ea1SDimitry Andric         switch (Ops[0].getSimpleValueType().SimpleTy) {
2376*0fca6ea1SDimitry Andric         default:
23770b57cec5SDimitry Andric           break;
2378*0fca6ea1SDimitry Andric         case MVT::i32:
2379*0fca6ea1SDimitry Andric           Opcode = NVPTX::StoreParamI8TruncI32_r;
23800b57cec5SDimitry Andric           break;
2381*0fca6ea1SDimitry Andric         case MVT::i64:
2382*0fca6ea1SDimitry Andric           Opcode = NVPTX::StoreParamI8TruncI64_r;
23830b57cec5SDimitry Andric           break;
23840b57cec5SDimitry Andric         }
2385*0fca6ea1SDimitry Andric       }
2386*0fca6ea1SDimitry Andric       break;
2387*0fca6ea1SDimitry Andric     }
2388*0fca6ea1SDimitry Andric     case 2:
2389*0fca6ea1SDimitry Andric     case 4: {
2390*0fca6ea1SDimitry Andric       MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
2391*0fca6ea1SDimitry Andric       Opcode = pickOpcodeForVectorStParam(Ops, NumElts, MemTy, CurDAG, DL);
2392*0fca6ea1SDimitry Andric       break;
2393*0fca6ea1SDimitry Andric     }
2394*0fca6ea1SDimitry Andric     }
23950b57cec5SDimitry Andric     break;
23960b57cec5SDimitry Andric   // Special case: if we have a sign-extend/zero-extend node, insert the
23970b57cec5SDimitry Andric   // conversion instruction first, and use that as the value operand to
23980b57cec5SDimitry Andric   // the selected StoreParam node.
23990b57cec5SDimitry Andric   case NVPTXISD::StoreParamU32: {
2400*0fca6ea1SDimitry Andric     Opcode = NVPTX::StoreParamI32_r;
24010b57cec5SDimitry Andric     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
24020b57cec5SDimitry Andric                                                 MVT::i32);
24030b57cec5SDimitry Andric     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
24040b57cec5SDimitry Andric                                          MVT::i32, Ops[0], CvtNone);
24050b57cec5SDimitry Andric     Ops[0] = SDValue(Cvt, 0);
24060b57cec5SDimitry Andric     break;
24070b57cec5SDimitry Andric   }
24080b57cec5SDimitry Andric   case NVPTXISD::StoreParamS32: {
2409*0fca6ea1SDimitry Andric     Opcode = NVPTX::StoreParamI32_r;
24100b57cec5SDimitry Andric     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
24110b57cec5SDimitry Andric                                                 MVT::i32);
24120b57cec5SDimitry Andric     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
24130b57cec5SDimitry Andric                                          MVT::i32, Ops[0], CvtNone);
24140b57cec5SDimitry Andric     Ops[0] = SDValue(Cvt, 0);
24150b57cec5SDimitry Andric     break;
24160b57cec5SDimitry Andric   }
24170b57cec5SDimitry Andric   }
24180b57cec5SDimitry Andric 
24190b57cec5SDimitry Andric   SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
242081ad6265SDimitry Andric   SDNode *Ret = CurDAG->getMachineNode(*Opcode, DL, RetVTs, Ops);
24210b57cec5SDimitry Andric   MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
24220b57cec5SDimitry Andric   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ret), {MemRef});
24230b57cec5SDimitry Andric 
24240b57cec5SDimitry Andric   ReplaceNode(N, Ret);
24250b57cec5SDimitry Andric   return true;
24260b57cec5SDimitry Andric }
24270b57cec5SDimitry Andric 
24280b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryTextureIntrinsic(SDNode *N) {
24290b57cec5SDimitry Andric   unsigned Opc = 0;
24300b57cec5SDimitry Andric 
24310b57cec5SDimitry Andric   switch (N->getOpcode()) {
24320b57cec5SDimitry Andric   default: return false;
24330b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatS32:
2434349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_F32_S32_RR;
24350b57cec5SDimitry Andric     break;
24360b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatFloat:
2437349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_F32_F32_RR;
24380b57cec5SDimitry Andric     break;
24390b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatFloatLevel:
2440349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_F32_F32_LEVEL_RR;
24410b57cec5SDimitry Andric     break;
24420b57cec5SDimitry Andric   case NVPTXISD::Tex1DFloatFloatGrad:
2443349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_F32_F32_GRAD_RR;
24440b57cec5SDimitry Andric     break;
24450b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32S32:
2446349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_S32_S32_RR;
24470b57cec5SDimitry Andric     break;
24480b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32Float:
2449349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_S32_F32_RR;
24500b57cec5SDimitry Andric     break;
24510b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32FloatLevel:
2452349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_S32_F32_LEVEL_RR;
24530b57cec5SDimitry Andric     break;
24540b57cec5SDimitry Andric   case NVPTXISD::Tex1DS32FloatGrad:
2455349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_S32_F32_GRAD_RR;
24560b57cec5SDimitry Andric     break;
24570b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32S32:
2458349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_U32_S32_RR;
24590b57cec5SDimitry Andric     break;
24600b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32Float:
2461349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_U32_F32_RR;
24620b57cec5SDimitry Andric     break;
24630b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32FloatLevel:
2464349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_U32_F32_LEVEL_RR;
24650b57cec5SDimitry Andric     break;
24660b57cec5SDimitry Andric   case NVPTXISD::Tex1DU32FloatGrad:
2467349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_U32_F32_GRAD_RR;
24680b57cec5SDimitry Andric     break;
24690b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatS32:
2470349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_F32_S32_RR;
24710b57cec5SDimitry Andric     break;
24720b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatFloat:
2473349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_RR;
24740b57cec5SDimitry Andric     break;
24750b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatFloatLevel:
2476349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL_RR;
24770b57cec5SDimitry Andric     break;
24780b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayFloatFloatGrad:
2479349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD_RR;
24800b57cec5SDimitry Andric     break;
24810b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32S32:
2482349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_S32_S32_RR;
24830b57cec5SDimitry Andric     break;
24840b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32Float:
2485349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_RR;
24860b57cec5SDimitry Andric     break;
24870b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32FloatLevel:
2488349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL_RR;
24890b57cec5SDimitry Andric     break;
24900b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayS32FloatGrad:
2491349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD_RR;
24920b57cec5SDimitry Andric     break;
24930b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32S32:
2494349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_U32_S32_RR;
24950b57cec5SDimitry Andric     break;
24960b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32Float:
2497349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_RR;
24980b57cec5SDimitry Andric     break;
24990b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32FloatLevel:
2500349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL_RR;
25010b57cec5SDimitry Andric     break;
25020b57cec5SDimitry Andric   case NVPTXISD::Tex1DArrayU32FloatGrad:
2503349cc55cSDimitry Andric     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD_RR;
25040b57cec5SDimitry Andric     break;
25050b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatS32:
2506349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_F32_S32_RR;
25070b57cec5SDimitry Andric     break;
25080b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatFloat:
2509349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_F32_F32_RR;
25100b57cec5SDimitry Andric     break;
25110b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatFloatLevel:
2512349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_F32_F32_LEVEL_RR;
25130b57cec5SDimitry Andric     break;
25140b57cec5SDimitry Andric   case NVPTXISD::Tex2DFloatFloatGrad:
2515349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_F32_F32_GRAD_RR;
25160b57cec5SDimitry Andric     break;
25170b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32S32:
2518349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_S32_S32_RR;
25190b57cec5SDimitry Andric     break;
25200b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32Float:
2521349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_S32_F32_RR;
25220b57cec5SDimitry Andric     break;
25230b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32FloatLevel:
2524349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_S32_F32_LEVEL_RR;
25250b57cec5SDimitry Andric     break;
25260b57cec5SDimitry Andric   case NVPTXISD::Tex2DS32FloatGrad:
2527349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_S32_F32_GRAD_RR;
25280b57cec5SDimitry Andric     break;
25290b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32S32:
2530349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_U32_S32_RR;
25310b57cec5SDimitry Andric     break;
25320b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32Float:
2533349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_U32_F32_RR;
25340b57cec5SDimitry Andric     break;
25350b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32FloatLevel:
2536349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_U32_F32_LEVEL_RR;
25370b57cec5SDimitry Andric     break;
25380b57cec5SDimitry Andric   case NVPTXISD::Tex2DU32FloatGrad:
2539349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_U32_F32_GRAD_RR;
25400b57cec5SDimitry Andric     break;
25410b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatS32:
2542349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_F32_S32_RR;
25430b57cec5SDimitry Andric     break;
25440b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatFloat:
2545349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_RR;
25460b57cec5SDimitry Andric     break;
25470b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatFloatLevel:
2548349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL_RR;
25490b57cec5SDimitry Andric     break;
25500b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayFloatFloatGrad:
2551349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD_RR;
25520b57cec5SDimitry Andric     break;
25530b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32S32:
2554349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_S32_S32_RR;
25550b57cec5SDimitry Andric     break;
25560b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32Float:
2557349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_RR;
25580b57cec5SDimitry Andric     break;
25590b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32FloatLevel:
2560349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL_RR;
25610b57cec5SDimitry Andric     break;
25620b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayS32FloatGrad:
2563349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD_RR;
25640b57cec5SDimitry Andric     break;
25650b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32S32:
2566349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_U32_S32_RR;
25670b57cec5SDimitry Andric     break;
25680b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32Float:
2569349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_RR;
25700b57cec5SDimitry Andric     break;
25710b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32FloatLevel:
2572349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL_RR;
25730b57cec5SDimitry Andric     break;
25740b57cec5SDimitry Andric   case NVPTXISD::Tex2DArrayU32FloatGrad:
2575349cc55cSDimitry Andric     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD_RR;
25760b57cec5SDimitry Andric     break;
25770b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatS32:
2578349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_F32_S32_RR;
25790b57cec5SDimitry Andric     break;
25800b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatFloat:
2581349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_F32_F32_RR;
25820b57cec5SDimitry Andric     break;
25830b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatFloatLevel:
2584349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_F32_F32_LEVEL_RR;
25850b57cec5SDimitry Andric     break;
25860b57cec5SDimitry Andric   case NVPTXISD::Tex3DFloatFloatGrad:
2587349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_F32_F32_GRAD_RR;
25880b57cec5SDimitry Andric     break;
25890b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32S32:
2590349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_S32_S32_RR;
25910b57cec5SDimitry Andric     break;
25920b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32Float:
2593349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_S32_F32_RR;
25940b57cec5SDimitry Andric     break;
25950b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32FloatLevel:
2596349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_S32_F32_LEVEL_RR;
25970b57cec5SDimitry Andric     break;
25980b57cec5SDimitry Andric   case NVPTXISD::Tex3DS32FloatGrad:
2599349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_S32_F32_GRAD_RR;
26000b57cec5SDimitry Andric     break;
26010b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32S32:
2602349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_U32_S32_RR;
26030b57cec5SDimitry Andric     break;
26040b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32Float:
2605349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_U32_F32_RR;
26060b57cec5SDimitry Andric     break;
26070b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32FloatLevel:
2608349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_U32_F32_LEVEL_RR;
26090b57cec5SDimitry Andric     break;
26100b57cec5SDimitry Andric   case NVPTXISD::Tex3DU32FloatGrad:
2611349cc55cSDimitry Andric     Opc = NVPTX::TEX_3D_U32_F32_GRAD_RR;
26120b57cec5SDimitry Andric     break;
26130b57cec5SDimitry Andric   case NVPTXISD::TexCubeFloatFloat:
2614349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_F32_F32_RR;
26150b57cec5SDimitry Andric     break;
26160b57cec5SDimitry Andric   case NVPTXISD::TexCubeFloatFloatLevel:
2617349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL_RR;
26180b57cec5SDimitry Andric     break;
26190b57cec5SDimitry Andric   case NVPTXISD::TexCubeS32Float:
2620349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_S32_F32_RR;
26210b57cec5SDimitry Andric     break;
26220b57cec5SDimitry Andric   case NVPTXISD::TexCubeS32FloatLevel:
2623349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL_RR;
26240b57cec5SDimitry Andric     break;
26250b57cec5SDimitry Andric   case NVPTXISD::TexCubeU32Float:
2626349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_U32_F32_RR;
26270b57cec5SDimitry Andric     break;
26280b57cec5SDimitry Andric   case NVPTXISD::TexCubeU32FloatLevel:
2629349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL_RR;
26300b57cec5SDimitry Andric     break;
26310b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayFloatFloat:
2632349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_RR;
26330b57cec5SDimitry Andric     break;
26340b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayFloatFloatLevel:
2635349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL_RR;
26360b57cec5SDimitry Andric     break;
26370b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayS32Float:
2638349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_RR;
26390b57cec5SDimitry Andric     break;
26400b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayS32FloatLevel:
2641349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL_RR;
26420b57cec5SDimitry Andric     break;
26430b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayU32Float:
2644349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_RR;
26450b57cec5SDimitry Andric     break;
26460b57cec5SDimitry Andric   case NVPTXISD::TexCubeArrayU32FloatLevel:
2647349cc55cSDimitry Andric     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL_RR;
26480b57cec5SDimitry Andric     break;
26490b57cec5SDimitry Andric   case NVPTXISD::Tld4R2DFloatFloat:
2650349cc55cSDimitry Andric     Opc = NVPTX::TLD4_R_2D_F32_F32_RR;
26510b57cec5SDimitry Andric     break;
26520b57cec5SDimitry Andric   case NVPTXISD::Tld4G2DFloatFloat:
2653349cc55cSDimitry Andric     Opc = NVPTX::TLD4_G_2D_F32_F32_RR;
26540b57cec5SDimitry Andric     break;
26550b57cec5SDimitry Andric   case NVPTXISD::Tld4B2DFloatFloat:
2656349cc55cSDimitry Andric     Opc = NVPTX::TLD4_B_2D_F32_F32_RR;
26570b57cec5SDimitry Andric     break;
26580b57cec5SDimitry Andric   case NVPTXISD::Tld4A2DFloatFloat:
2659349cc55cSDimitry Andric     Opc = NVPTX::TLD4_A_2D_F32_F32_RR;
26600b57cec5SDimitry Andric     break;
26610b57cec5SDimitry Andric   case NVPTXISD::Tld4R2DS64Float:
2662349cc55cSDimitry Andric     Opc = NVPTX::TLD4_R_2D_S32_F32_RR;
26630b57cec5SDimitry Andric     break;
26640b57cec5SDimitry Andric   case NVPTXISD::Tld4G2DS64Float:
2665349cc55cSDimitry Andric     Opc = NVPTX::TLD4_G_2D_S32_F32_RR;
26660b57cec5SDimitry Andric     break;
26670b57cec5SDimitry Andric   case NVPTXISD::Tld4B2DS64Float:
2668349cc55cSDimitry Andric     Opc = NVPTX::TLD4_B_2D_S32_F32_RR;
26690b57cec5SDimitry Andric     break;
26700b57cec5SDimitry Andric   case NVPTXISD::Tld4A2DS64Float:
2671349cc55cSDimitry Andric     Opc = NVPTX::TLD4_A_2D_S32_F32_RR;
26720b57cec5SDimitry Andric     break;
26730b57cec5SDimitry Andric   case NVPTXISD::Tld4R2DU64Float:
2674349cc55cSDimitry Andric     Opc = NVPTX::TLD4_R_2D_U32_F32_RR;
26750b57cec5SDimitry Andric     break;
26760b57cec5SDimitry Andric   case NVPTXISD::Tld4G2DU64Float:
2677349cc55cSDimitry Andric     Opc = NVPTX::TLD4_G_2D_U32_F32_RR;
26780b57cec5SDimitry Andric     break;
26790b57cec5SDimitry Andric   case NVPTXISD::Tld4B2DU64Float:
2680349cc55cSDimitry Andric     Opc = NVPTX::TLD4_B_2D_U32_F32_RR;
26810b57cec5SDimitry Andric     break;
26820b57cec5SDimitry Andric   case NVPTXISD::Tld4A2DU64Float:
2683349cc55cSDimitry Andric     Opc = NVPTX::TLD4_A_2D_U32_F32_RR;
26840b57cec5SDimitry Andric     break;
26850b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatS32:
2686349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_F32_S32_R;
26870b57cec5SDimitry Andric     break;
26880b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatFloat:
2689349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_R;
26900b57cec5SDimitry Andric     break;
26910b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatFloatLevel:
2692349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL_R;
26930b57cec5SDimitry Andric     break;
26940b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DFloatFloatGrad:
2695349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD_R;
26960b57cec5SDimitry Andric     break;
26970b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32S32:
2698349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_S32_S32_R;
26990b57cec5SDimitry Andric     break;
27000b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32Float:
2701349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_R;
27020b57cec5SDimitry Andric     break;
27030b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32FloatLevel:
2704349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL_R;
27050b57cec5SDimitry Andric     break;
27060b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DS32FloatGrad:
2707349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD_R;
27080b57cec5SDimitry Andric     break;
27090b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32S32:
2710349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_U32_S32_R;
27110b57cec5SDimitry Andric     break;
27120b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32Float:
2713349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_R;
27140b57cec5SDimitry Andric     break;
27150b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32FloatLevel:
2716349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL_R;
27170b57cec5SDimitry Andric     break;
27180b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DU32FloatGrad:
2719349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD_R;
27200b57cec5SDimitry Andric     break;
27210b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatS32:
2722349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32_R;
27230b57cec5SDimitry Andric     break;
27240b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatFloat:
2725349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_R;
27260b57cec5SDimitry Andric     break;
27270b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
2728349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL_R;
27290b57cec5SDimitry Andric     break;
27300b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
2731349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD_R;
27320b57cec5SDimitry Andric     break;
27330b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32S32:
2734349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32_R;
27350b57cec5SDimitry Andric     break;
27360b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32Float:
2737349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_R;
27380b57cec5SDimitry Andric     break;
27390b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
2740349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL_R;
27410b57cec5SDimitry Andric     break;
27420b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
2743349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD_R;
27440b57cec5SDimitry Andric     break;
27450b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32S32:
2746349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32_R;
27470b57cec5SDimitry Andric     break;
27480b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32Float:
2749349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_R;
27500b57cec5SDimitry Andric     break;
27510b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
2752349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL_R;
27530b57cec5SDimitry Andric     break;
27540b57cec5SDimitry Andric   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
2755349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD_R;
27560b57cec5SDimitry Andric     break;
27570b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatS32:
2758349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_F32_S32_R;
27590b57cec5SDimitry Andric     break;
27600b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatFloat:
2761349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_R;
27620b57cec5SDimitry Andric     break;
27630b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatFloatLevel:
2764349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL_R;
27650b57cec5SDimitry Andric     break;
27660b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DFloatFloatGrad:
2767349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD_R;
27680b57cec5SDimitry Andric     break;
27690b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32S32:
2770349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_S32_S32_R;
27710b57cec5SDimitry Andric     break;
27720b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32Float:
2773349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_R;
27740b57cec5SDimitry Andric     break;
27750b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32FloatLevel:
2776349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL_R;
27770b57cec5SDimitry Andric     break;
27780b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DS32FloatGrad:
2779349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD_R;
27800b57cec5SDimitry Andric     break;
27810b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32S32:
2782349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_U32_S32_R;
27830b57cec5SDimitry Andric     break;
27840b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32Float:
2785349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_R;
27860b57cec5SDimitry Andric     break;
27870b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32FloatLevel:
2788349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL_R;
27890b57cec5SDimitry Andric     break;
27900b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DU32FloatGrad:
2791349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD_R;
27920b57cec5SDimitry Andric     break;
27930b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatS32:
2794349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32_R;
27950b57cec5SDimitry Andric     break;
27960b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatFloat:
2797349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_R;
27980b57cec5SDimitry Andric     break;
27990b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
2800349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL_R;
28010b57cec5SDimitry Andric     break;
28020b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
2803349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD_R;
28040b57cec5SDimitry Andric     break;
28050b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32S32:
2806349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32_R;
28070b57cec5SDimitry Andric     break;
28080b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32Float:
2809349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_R;
28100b57cec5SDimitry Andric     break;
28110b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
2812349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL_R;
28130b57cec5SDimitry Andric     break;
28140b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
2815349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD_R;
28160b57cec5SDimitry Andric     break;
28170b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32S32:
2818349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32_R;
28190b57cec5SDimitry Andric     break;
28200b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32Float:
2821349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_R;
28220b57cec5SDimitry Andric     break;
28230b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
2824349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL_R;
28250b57cec5SDimitry Andric     break;
28260b57cec5SDimitry Andric   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
2827349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD_R;
28280b57cec5SDimitry Andric     break;
28290b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatS32:
2830349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_F32_S32_R;
28310b57cec5SDimitry Andric     break;
28320b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatFloat:
2833349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_R;
28340b57cec5SDimitry Andric     break;
28350b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatFloatLevel:
2836349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL_R;
28370b57cec5SDimitry Andric     break;
28380b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DFloatFloatGrad:
2839349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD_R;
28400b57cec5SDimitry Andric     break;
28410b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32S32:
2842349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_S32_S32_R;
28430b57cec5SDimitry Andric     break;
28440b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32Float:
2845349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_R;
28460b57cec5SDimitry Andric     break;
28470b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32FloatLevel:
2848349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL_R;
28490b57cec5SDimitry Andric     break;
28500b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DS32FloatGrad:
2851349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD_R;
28520b57cec5SDimitry Andric     break;
28530b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32S32:
2854349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_U32_S32_R;
28550b57cec5SDimitry Andric     break;
28560b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32Float:
2857349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_R;
28580b57cec5SDimitry Andric     break;
28590b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32FloatLevel:
2860349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL_R;
28610b57cec5SDimitry Andric     break;
28620b57cec5SDimitry Andric   case NVPTXISD::TexUnified3DU32FloatGrad:
2863349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD_R;
28640b57cec5SDimitry Andric     break;
28650b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeFloatFloat:
2866349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_R;
28670b57cec5SDimitry Andric     break;
28680b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
2869349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL_R;
28700b57cec5SDimitry Andric     break;
28710b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeS32Float:
2872349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_R;
28730b57cec5SDimitry Andric     break;
28740b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
2875349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL_R;
28760b57cec5SDimitry Andric     break;
28770b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeU32Float:
2878349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_R;
28790b57cec5SDimitry Andric     break;
28800b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
2881349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL_R;
28820b57cec5SDimitry Andric     break;
28830b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
2884349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_R;
28850b57cec5SDimitry Andric     break;
28860b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
2887349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL_R;
28880b57cec5SDimitry Andric     break;
28890b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayS32Float:
2890349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_R;
28910b57cec5SDimitry Andric     break;
28920b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
2893349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL_R;
28940b57cec5SDimitry Andric     break;
28950b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayU32Float:
2896349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_R;
28970b57cec5SDimitry Andric     break;
28980b57cec5SDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
2899349cc55cSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL_R;
29000b57cec5SDimitry Andric     break;
29010b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
2902349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32_R;
29030b57cec5SDimitry Andric     break;
29040b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
2905349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32_R;
29060b57cec5SDimitry Andric     break;
29070b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
2908349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32_R;
29090b57cec5SDimitry Andric     break;
29100b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
2911349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32_R;
29120b57cec5SDimitry Andric     break;
29130b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedR2DS64Float:
2914349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32_R;
29150b57cec5SDimitry Andric     break;
29160b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedG2DS64Float:
2917349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32_R;
29180b57cec5SDimitry Andric     break;
29190b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedB2DS64Float:
2920349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32_R;
29210b57cec5SDimitry Andric     break;
29220b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedA2DS64Float:
2923349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32_R;
29240b57cec5SDimitry Andric     break;
29250b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedR2DU64Float:
2926349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32_R;
29270b57cec5SDimitry Andric     break;
29280b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedG2DU64Float:
2929349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32_R;
29300b57cec5SDimitry Andric     break;
29310b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedB2DU64Float:
2932349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32_R;
29330b57cec5SDimitry Andric     break;
29340b57cec5SDimitry Andric   case NVPTXISD::Tld4UnifiedA2DU64Float:
2935349cc55cSDimitry Andric     Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32_R;
29360b57cec5SDimitry Andric     break;
29377a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeFloatFloatGrad:
29387a6dacacSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_GRAD_R;
29397a6dacacSDimitry Andric     break;
29407a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeS32FloatGrad:
29417a6dacacSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_GRAD_R;
29427a6dacacSDimitry Andric     break;
29437a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeU32FloatGrad:
29447a6dacacSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_GRAD_R;
29457a6dacacSDimitry Andric     break;
29467a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayFloatFloatGrad:
29477a6dacacSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_GRAD_R;
29487a6dacacSDimitry Andric     break;
29497a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayS32FloatGrad:
29507a6dacacSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_GRAD_R;
29517a6dacacSDimitry Andric     break;
29527a6dacacSDimitry Andric   case NVPTXISD::TexUnifiedCubeArrayU32FloatGrad:
29537a6dacacSDimitry Andric     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_GRAD_R;
29547a6dacacSDimitry Andric     break;
29550b57cec5SDimitry Andric   }
29560b57cec5SDimitry Andric 
29570b57cec5SDimitry Andric   // Copy over operands
2958e8d8bef9SDimitry Andric   SmallVector<SDValue, 8> Ops(drop_begin(N->ops()));
29590b57cec5SDimitry Andric   Ops.push_back(N->getOperand(0)); // Move chain to the back.
29600b57cec5SDimitry Andric 
29610b57cec5SDimitry Andric   ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops));
29620b57cec5SDimitry Andric   return true;
29630b57cec5SDimitry Andric }
29640b57cec5SDimitry Andric 
29650b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::trySurfaceIntrinsic(SDNode *N) {
29660b57cec5SDimitry Andric   unsigned Opc = 0;
29670b57cec5SDimitry Andric   switch (N->getOpcode()) {
29680b57cec5SDimitry Andric   default: return false;
29690b57cec5SDimitry Andric   case NVPTXISD::Suld1DI8Clamp:
2970349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I8_CLAMP_R;
29710b57cec5SDimitry Andric     break;
29720b57cec5SDimitry Andric   case NVPTXISD::Suld1DI16Clamp:
2973349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I16_CLAMP_R;
29740b57cec5SDimitry Andric     break;
29750b57cec5SDimitry Andric   case NVPTXISD::Suld1DI32Clamp:
2976349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I32_CLAMP_R;
29770b57cec5SDimitry Andric     break;
29780b57cec5SDimitry Andric   case NVPTXISD::Suld1DI64Clamp:
2979349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I64_CLAMP_R;
29800b57cec5SDimitry Andric     break;
29810b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I8Clamp:
2982349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I8_CLAMP_R;
29830b57cec5SDimitry Andric     break;
29840b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I16Clamp:
2985349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I16_CLAMP_R;
29860b57cec5SDimitry Andric     break;
29870b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I32Clamp:
2988349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I32_CLAMP_R;
29890b57cec5SDimitry Andric     break;
29900b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I64Clamp:
2991349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I64_CLAMP_R;
29920b57cec5SDimitry Andric     break;
29930b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I8Clamp:
2994349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I8_CLAMP_R;
29950b57cec5SDimitry Andric     break;
29960b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I16Clamp:
2997349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I16_CLAMP_R;
29980b57cec5SDimitry Andric     break;
29990b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I32Clamp:
3000349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I32_CLAMP_R;
30010b57cec5SDimitry Andric     break;
30020b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI8Clamp:
3003349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP_R;
30040b57cec5SDimitry Andric     break;
30050b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI16Clamp:
3006349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP_R;
30070b57cec5SDimitry Andric     break;
30080b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI32Clamp:
3009349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP_R;
30100b57cec5SDimitry Andric     break;
30110b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI64Clamp:
3012349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP_R;
30130b57cec5SDimitry Andric     break;
30140b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I8Clamp:
3015349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP_R;
30160b57cec5SDimitry Andric     break;
30170b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I16Clamp:
3018349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP_R;
30190b57cec5SDimitry Andric     break;
30200b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I32Clamp:
3021349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP_R;
30220b57cec5SDimitry Andric     break;
30230b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I64Clamp:
3024349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP_R;
30250b57cec5SDimitry Andric     break;
30260b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I8Clamp:
3027349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP_R;
30280b57cec5SDimitry Andric     break;
30290b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I16Clamp:
3030349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP_R;
30310b57cec5SDimitry Andric     break;
30320b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I32Clamp:
3033349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP_R;
30340b57cec5SDimitry Andric     break;
30350b57cec5SDimitry Andric   case NVPTXISD::Suld2DI8Clamp:
3036349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I8_CLAMP_R;
30370b57cec5SDimitry Andric     break;
30380b57cec5SDimitry Andric   case NVPTXISD::Suld2DI16Clamp:
3039349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I16_CLAMP_R;
30400b57cec5SDimitry Andric     break;
30410b57cec5SDimitry Andric   case NVPTXISD::Suld2DI32Clamp:
3042349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I32_CLAMP_R;
30430b57cec5SDimitry Andric     break;
30440b57cec5SDimitry Andric   case NVPTXISD::Suld2DI64Clamp:
3045349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I64_CLAMP_R;
30460b57cec5SDimitry Andric     break;
30470b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I8Clamp:
3048349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I8_CLAMP_R;
30490b57cec5SDimitry Andric     break;
30500b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I16Clamp:
3051349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I16_CLAMP_R;
30520b57cec5SDimitry Andric     break;
30530b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I32Clamp:
3054349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I32_CLAMP_R;
30550b57cec5SDimitry Andric     break;
30560b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I64Clamp:
3057349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I64_CLAMP_R;
30580b57cec5SDimitry Andric     break;
30590b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I8Clamp:
3060349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I8_CLAMP_R;
30610b57cec5SDimitry Andric     break;
30620b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I16Clamp:
3063349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I16_CLAMP_R;
30640b57cec5SDimitry Andric     break;
30650b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I32Clamp:
3066349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I32_CLAMP_R;
30670b57cec5SDimitry Andric     break;
30680b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI8Clamp:
3069349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP_R;
30700b57cec5SDimitry Andric     break;
30710b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI16Clamp:
3072349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP_R;
30730b57cec5SDimitry Andric     break;
30740b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI32Clamp:
3075349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP_R;
30760b57cec5SDimitry Andric     break;
30770b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI64Clamp:
3078349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP_R;
30790b57cec5SDimitry Andric     break;
30800b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I8Clamp:
3081349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP_R;
30820b57cec5SDimitry Andric     break;
30830b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I16Clamp:
3084349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP_R;
30850b57cec5SDimitry Andric     break;
30860b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I32Clamp:
3087349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP_R;
30880b57cec5SDimitry Andric     break;
30890b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I64Clamp:
3090349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP_R;
30910b57cec5SDimitry Andric     break;
30920b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I8Clamp:
3093349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP_R;
30940b57cec5SDimitry Andric     break;
30950b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I16Clamp:
3096349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP_R;
30970b57cec5SDimitry Andric     break;
30980b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I32Clamp:
3099349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP_R;
31000b57cec5SDimitry Andric     break;
31010b57cec5SDimitry Andric   case NVPTXISD::Suld3DI8Clamp:
3102349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I8_CLAMP_R;
31030b57cec5SDimitry Andric     break;
31040b57cec5SDimitry Andric   case NVPTXISD::Suld3DI16Clamp:
3105349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I16_CLAMP_R;
31060b57cec5SDimitry Andric     break;
31070b57cec5SDimitry Andric   case NVPTXISD::Suld3DI32Clamp:
3108349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I32_CLAMP_R;
31090b57cec5SDimitry Andric     break;
31100b57cec5SDimitry Andric   case NVPTXISD::Suld3DI64Clamp:
3111349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I64_CLAMP_R;
31120b57cec5SDimitry Andric     break;
31130b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I8Clamp:
3114349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I8_CLAMP_R;
31150b57cec5SDimitry Andric     break;
31160b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I16Clamp:
3117349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I16_CLAMP_R;
31180b57cec5SDimitry Andric     break;
31190b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I32Clamp:
3120349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I32_CLAMP_R;
31210b57cec5SDimitry Andric     break;
31220b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I64Clamp:
3123349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I64_CLAMP_R;
31240b57cec5SDimitry Andric     break;
31250b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I8Clamp:
3126349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I8_CLAMP_R;
31270b57cec5SDimitry Andric     break;
31280b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I16Clamp:
3129349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I16_CLAMP_R;
31300b57cec5SDimitry Andric     break;
31310b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I32Clamp:
3132349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I32_CLAMP_R;
31330b57cec5SDimitry Andric     break;
31340b57cec5SDimitry Andric   case NVPTXISD::Suld1DI8Trap:
3135349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I8_TRAP_R;
31360b57cec5SDimitry Andric     break;
31370b57cec5SDimitry Andric   case NVPTXISD::Suld1DI16Trap:
3138349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I16_TRAP_R;
31390b57cec5SDimitry Andric     break;
31400b57cec5SDimitry Andric   case NVPTXISD::Suld1DI32Trap:
3141349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I32_TRAP_R;
31420b57cec5SDimitry Andric     break;
31430b57cec5SDimitry Andric   case NVPTXISD::Suld1DI64Trap:
3144349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I64_TRAP_R;
31450b57cec5SDimitry Andric     break;
31460b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I8Trap:
3147349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I8_TRAP_R;
31480b57cec5SDimitry Andric     break;
31490b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I16Trap:
3150349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I16_TRAP_R;
31510b57cec5SDimitry Andric     break;
31520b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I32Trap:
3153349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I32_TRAP_R;
31540b57cec5SDimitry Andric     break;
31550b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I64Trap:
3156349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I64_TRAP_R;
31570b57cec5SDimitry Andric     break;
31580b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I8Trap:
3159349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I8_TRAP_R;
31600b57cec5SDimitry Andric     break;
31610b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I16Trap:
3162349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I16_TRAP_R;
31630b57cec5SDimitry Andric     break;
31640b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I32Trap:
3165349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I32_TRAP_R;
31660b57cec5SDimitry Andric     break;
31670b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI8Trap:
3168349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP_R;
31690b57cec5SDimitry Andric     break;
31700b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI16Trap:
3171349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP_R;
31720b57cec5SDimitry Andric     break;
31730b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI32Trap:
3174349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP_R;
31750b57cec5SDimitry Andric     break;
31760b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI64Trap:
3177349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP_R;
31780b57cec5SDimitry Andric     break;
31790b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I8Trap:
3180349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP_R;
31810b57cec5SDimitry Andric     break;
31820b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I16Trap:
3183349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP_R;
31840b57cec5SDimitry Andric     break;
31850b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I32Trap:
3186349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP_R;
31870b57cec5SDimitry Andric     break;
31880b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I64Trap:
3189349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP_R;
31900b57cec5SDimitry Andric     break;
31910b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I8Trap:
3192349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP_R;
31930b57cec5SDimitry Andric     break;
31940b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I16Trap:
3195349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP_R;
31960b57cec5SDimitry Andric     break;
31970b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I32Trap:
3198349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP_R;
31990b57cec5SDimitry Andric     break;
32000b57cec5SDimitry Andric   case NVPTXISD::Suld2DI8Trap:
3201349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I8_TRAP_R;
32020b57cec5SDimitry Andric     break;
32030b57cec5SDimitry Andric   case NVPTXISD::Suld2DI16Trap:
3204349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I16_TRAP_R;
32050b57cec5SDimitry Andric     break;
32060b57cec5SDimitry Andric   case NVPTXISD::Suld2DI32Trap:
3207349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I32_TRAP_R;
32080b57cec5SDimitry Andric     break;
32090b57cec5SDimitry Andric   case NVPTXISD::Suld2DI64Trap:
3210349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I64_TRAP_R;
32110b57cec5SDimitry Andric     break;
32120b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I8Trap:
3213349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I8_TRAP_R;
32140b57cec5SDimitry Andric     break;
32150b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I16Trap:
3216349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I16_TRAP_R;
32170b57cec5SDimitry Andric     break;
32180b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I32Trap:
3219349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I32_TRAP_R;
32200b57cec5SDimitry Andric     break;
32210b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I64Trap:
3222349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I64_TRAP_R;
32230b57cec5SDimitry Andric     break;
32240b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I8Trap:
3225349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I8_TRAP_R;
32260b57cec5SDimitry Andric     break;
32270b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I16Trap:
3228349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I16_TRAP_R;
32290b57cec5SDimitry Andric     break;
32300b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I32Trap:
3231349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I32_TRAP_R;
32320b57cec5SDimitry Andric     break;
32330b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI8Trap:
3234349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP_R;
32350b57cec5SDimitry Andric     break;
32360b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI16Trap:
3237349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP_R;
32380b57cec5SDimitry Andric     break;
32390b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI32Trap:
3240349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP_R;
32410b57cec5SDimitry Andric     break;
32420b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI64Trap:
3243349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP_R;
32440b57cec5SDimitry Andric     break;
32450b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I8Trap:
3246349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP_R;
32470b57cec5SDimitry Andric     break;
32480b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I16Trap:
3249349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP_R;
32500b57cec5SDimitry Andric     break;
32510b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I32Trap:
3252349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP_R;
32530b57cec5SDimitry Andric     break;
32540b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I64Trap:
3255349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP_R;
32560b57cec5SDimitry Andric     break;
32570b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I8Trap:
3258349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP_R;
32590b57cec5SDimitry Andric     break;
32600b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I16Trap:
3261349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP_R;
32620b57cec5SDimitry Andric     break;
32630b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I32Trap:
3264349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP_R;
32650b57cec5SDimitry Andric     break;
32660b57cec5SDimitry Andric   case NVPTXISD::Suld3DI8Trap:
3267349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I8_TRAP_R;
32680b57cec5SDimitry Andric     break;
32690b57cec5SDimitry Andric   case NVPTXISD::Suld3DI16Trap:
3270349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I16_TRAP_R;
32710b57cec5SDimitry Andric     break;
32720b57cec5SDimitry Andric   case NVPTXISD::Suld3DI32Trap:
3273349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I32_TRAP_R;
32740b57cec5SDimitry Andric     break;
32750b57cec5SDimitry Andric   case NVPTXISD::Suld3DI64Trap:
3276349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I64_TRAP_R;
32770b57cec5SDimitry Andric     break;
32780b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I8Trap:
3279349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I8_TRAP_R;
32800b57cec5SDimitry Andric     break;
32810b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I16Trap:
3282349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I16_TRAP_R;
32830b57cec5SDimitry Andric     break;
32840b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I32Trap:
3285349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I32_TRAP_R;
32860b57cec5SDimitry Andric     break;
32870b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I64Trap:
3288349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I64_TRAP_R;
32890b57cec5SDimitry Andric     break;
32900b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I8Trap:
3291349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I8_TRAP_R;
32920b57cec5SDimitry Andric     break;
32930b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I16Trap:
3294349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I16_TRAP_R;
32950b57cec5SDimitry Andric     break;
32960b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I32Trap:
3297349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I32_TRAP_R;
32980b57cec5SDimitry Andric     break;
32990b57cec5SDimitry Andric   case NVPTXISD::Suld1DI8Zero:
3300349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I8_ZERO_R;
33010b57cec5SDimitry Andric     break;
33020b57cec5SDimitry Andric   case NVPTXISD::Suld1DI16Zero:
3303349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I16_ZERO_R;
33040b57cec5SDimitry Andric     break;
33050b57cec5SDimitry Andric   case NVPTXISD::Suld1DI32Zero:
3306349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I32_ZERO_R;
33070b57cec5SDimitry Andric     break;
33080b57cec5SDimitry Andric   case NVPTXISD::Suld1DI64Zero:
3309349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_I64_ZERO_R;
33100b57cec5SDimitry Andric     break;
33110b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I8Zero:
3312349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I8_ZERO_R;
33130b57cec5SDimitry Andric     break;
33140b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I16Zero:
3315349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I16_ZERO_R;
33160b57cec5SDimitry Andric     break;
33170b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I32Zero:
3318349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I32_ZERO_R;
33190b57cec5SDimitry Andric     break;
33200b57cec5SDimitry Andric   case NVPTXISD::Suld1DV2I64Zero:
3321349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V2I64_ZERO_R;
33220b57cec5SDimitry Andric     break;
33230b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I8Zero:
3324349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I8_ZERO_R;
33250b57cec5SDimitry Andric     break;
33260b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I16Zero:
3327349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I16_ZERO_R;
33280b57cec5SDimitry Andric     break;
33290b57cec5SDimitry Andric   case NVPTXISD::Suld1DV4I32Zero:
3330349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_V4I32_ZERO_R;
33310b57cec5SDimitry Andric     break;
33320b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI8Zero:
3333349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO_R;
33340b57cec5SDimitry Andric     break;
33350b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI16Zero:
3336349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO_R;
33370b57cec5SDimitry Andric     break;
33380b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI32Zero:
3339349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO_R;
33400b57cec5SDimitry Andric     break;
33410b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayI64Zero:
3342349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO_R;
33430b57cec5SDimitry Andric     break;
33440b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I8Zero:
3345349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO_R;
33460b57cec5SDimitry Andric     break;
33470b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I16Zero:
3348349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO_R;
33490b57cec5SDimitry Andric     break;
33500b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I32Zero:
3351349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO_R;
33520b57cec5SDimitry Andric     break;
33530b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV2I64Zero:
3354349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO_R;
33550b57cec5SDimitry Andric     break;
33560b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I8Zero:
3357349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO_R;
33580b57cec5SDimitry Andric     break;
33590b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I16Zero:
3360349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO_R;
33610b57cec5SDimitry Andric     break;
33620b57cec5SDimitry Andric   case NVPTXISD::Suld1DArrayV4I32Zero:
3363349cc55cSDimitry Andric     Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO_R;
33640b57cec5SDimitry Andric     break;
33650b57cec5SDimitry Andric   case NVPTXISD::Suld2DI8Zero:
3366349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I8_ZERO_R;
33670b57cec5SDimitry Andric     break;
33680b57cec5SDimitry Andric   case NVPTXISD::Suld2DI16Zero:
3369349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I16_ZERO_R;
33700b57cec5SDimitry Andric     break;
33710b57cec5SDimitry Andric   case NVPTXISD::Suld2DI32Zero:
3372349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I32_ZERO_R;
33730b57cec5SDimitry Andric     break;
33740b57cec5SDimitry Andric   case NVPTXISD::Suld2DI64Zero:
3375349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_I64_ZERO_R;
33760b57cec5SDimitry Andric     break;
33770b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I8Zero:
3378349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I8_ZERO_R;
33790b57cec5SDimitry Andric     break;
33800b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I16Zero:
3381349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I16_ZERO_R;
33820b57cec5SDimitry Andric     break;
33830b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I32Zero:
3384349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I32_ZERO_R;
33850b57cec5SDimitry Andric     break;
33860b57cec5SDimitry Andric   case NVPTXISD::Suld2DV2I64Zero:
3387349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V2I64_ZERO_R;
33880b57cec5SDimitry Andric     break;
33890b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I8Zero:
3390349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I8_ZERO_R;
33910b57cec5SDimitry Andric     break;
33920b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I16Zero:
3393349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I16_ZERO_R;
33940b57cec5SDimitry Andric     break;
33950b57cec5SDimitry Andric   case NVPTXISD::Suld2DV4I32Zero:
3396349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_V4I32_ZERO_R;
33970b57cec5SDimitry Andric     break;
33980b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI8Zero:
3399349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO_R;
34000b57cec5SDimitry Andric     break;
34010b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI16Zero:
3402349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO_R;
34030b57cec5SDimitry Andric     break;
34040b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI32Zero:
3405349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO_R;
34060b57cec5SDimitry Andric     break;
34070b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayI64Zero:
3408349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO_R;
34090b57cec5SDimitry Andric     break;
34100b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I8Zero:
3411349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO_R;
34120b57cec5SDimitry Andric     break;
34130b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I16Zero:
3414349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO_R;
34150b57cec5SDimitry Andric     break;
34160b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I32Zero:
3417349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO_R;
34180b57cec5SDimitry Andric     break;
34190b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV2I64Zero:
3420349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO_R;
34210b57cec5SDimitry Andric     break;
34220b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I8Zero:
3423349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO_R;
34240b57cec5SDimitry Andric     break;
34250b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I16Zero:
3426349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO_R;
34270b57cec5SDimitry Andric     break;
34280b57cec5SDimitry Andric   case NVPTXISD::Suld2DArrayV4I32Zero:
3429349cc55cSDimitry Andric     Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO_R;
34300b57cec5SDimitry Andric     break;
34310b57cec5SDimitry Andric   case NVPTXISD::Suld3DI8Zero:
3432349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I8_ZERO_R;
34330b57cec5SDimitry Andric     break;
34340b57cec5SDimitry Andric   case NVPTXISD::Suld3DI16Zero:
3435349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I16_ZERO_R;
34360b57cec5SDimitry Andric     break;
34370b57cec5SDimitry Andric   case NVPTXISD::Suld3DI32Zero:
3438349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I32_ZERO_R;
34390b57cec5SDimitry Andric     break;
34400b57cec5SDimitry Andric   case NVPTXISD::Suld3DI64Zero:
3441349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_I64_ZERO_R;
34420b57cec5SDimitry Andric     break;
34430b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I8Zero:
3444349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I8_ZERO_R;
34450b57cec5SDimitry Andric     break;
34460b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I16Zero:
3447349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I16_ZERO_R;
34480b57cec5SDimitry Andric     break;
34490b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I32Zero:
3450349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I32_ZERO_R;
34510b57cec5SDimitry Andric     break;
34520b57cec5SDimitry Andric   case NVPTXISD::Suld3DV2I64Zero:
3453349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V2I64_ZERO_R;
34540b57cec5SDimitry Andric     break;
34550b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I8Zero:
3456349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I8_ZERO_R;
34570b57cec5SDimitry Andric     break;
34580b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I16Zero:
3459349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I16_ZERO_R;
34600b57cec5SDimitry Andric     break;
34610b57cec5SDimitry Andric   case NVPTXISD::Suld3DV4I32Zero:
3462349cc55cSDimitry Andric     Opc = NVPTX::SULD_3D_V4I32_ZERO_R;
34630b57cec5SDimitry Andric     break;
34640b57cec5SDimitry Andric   }
34650b57cec5SDimitry Andric 
34660b57cec5SDimitry Andric   // Copy over operands
3467e8d8bef9SDimitry Andric   SmallVector<SDValue, 8> Ops(drop_begin(N->ops()));
34680b57cec5SDimitry Andric   Ops.push_back(N->getOperand(0)); // Move chain to the back.
34690b57cec5SDimitry Andric 
34700b57cec5SDimitry Andric   ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops));
34710b57cec5SDimitry Andric   return true;
34720b57cec5SDimitry Andric }
34730b57cec5SDimitry Andric 
34740b57cec5SDimitry Andric 
34750b57cec5SDimitry Andric /// SelectBFE - Look for instruction sequences that can be made more efficient
34760b57cec5SDimitry Andric /// by using the 'bfe' (bit-field extract) PTX instruction
34770b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::tryBFE(SDNode *N) {
34780b57cec5SDimitry Andric   SDLoc DL(N);
34790b57cec5SDimitry Andric   SDValue LHS = N->getOperand(0);
34800b57cec5SDimitry Andric   SDValue RHS = N->getOperand(1);
34810b57cec5SDimitry Andric   SDValue Len;
34820b57cec5SDimitry Andric   SDValue Start;
34830b57cec5SDimitry Andric   SDValue Val;
34840b57cec5SDimitry Andric   bool IsSigned = false;
34850b57cec5SDimitry Andric 
34860b57cec5SDimitry Andric   if (N->getOpcode() == ISD::AND) {
34870b57cec5SDimitry Andric     // Canonicalize the operands
34880b57cec5SDimitry Andric     // We want 'and %val, %mask'
34890b57cec5SDimitry Andric     if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
34900b57cec5SDimitry Andric       std::swap(LHS, RHS);
34910b57cec5SDimitry Andric     }
34920b57cec5SDimitry Andric 
34930b57cec5SDimitry Andric     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
34940b57cec5SDimitry Andric     if (!Mask) {
34950b57cec5SDimitry Andric       // We need a constant mask on the RHS of the AND
34960b57cec5SDimitry Andric       return false;
34970b57cec5SDimitry Andric     }
34980b57cec5SDimitry Andric 
34990b57cec5SDimitry Andric     // Extract the mask bits
35000b57cec5SDimitry Andric     uint64_t MaskVal = Mask->getZExtValue();
35010b57cec5SDimitry Andric     if (!isMask_64(MaskVal)) {
35020b57cec5SDimitry Andric       // We *could* handle shifted masks here, but doing so would require an
35030b57cec5SDimitry Andric       // 'and' operation to fix up the low-order bits so we would trade
35040b57cec5SDimitry Andric       // shr+and for bfe+and, which has the same throughput
35050b57cec5SDimitry Andric       return false;
35060b57cec5SDimitry Andric     }
35070b57cec5SDimitry Andric 
35080b57cec5SDimitry Andric     // How many bits are in our mask?
350906c3fb27SDimitry Andric     int64_t NumBits = countr_one(MaskVal);
35100b57cec5SDimitry Andric     Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
35110b57cec5SDimitry Andric 
35120b57cec5SDimitry Andric     if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
35130b57cec5SDimitry Andric       // We have a 'srl/and' pair, extract the effective start bit and length
35140b57cec5SDimitry Andric       Val = LHS.getNode()->getOperand(0);
35150b57cec5SDimitry Andric       Start = LHS.getNode()->getOperand(1);
35160b57cec5SDimitry Andric       ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
35170b57cec5SDimitry Andric       if (StartConst) {
35180b57cec5SDimitry Andric         uint64_t StartVal = StartConst->getZExtValue();
35190b57cec5SDimitry Andric         // How many "good" bits do we have left?  "good" is defined here as bits
35200b57cec5SDimitry Andric         // that exist in the original value, not shifted in.
352106c3fb27SDimitry Andric         int64_t GoodBits = Start.getValueSizeInBits() - StartVal;
35220b57cec5SDimitry Andric         if (NumBits > GoodBits) {
35230b57cec5SDimitry Andric           // Do not handle the case where bits have been shifted in. In theory
35240b57cec5SDimitry Andric           // we could handle this, but the cost is likely higher than just
35250b57cec5SDimitry Andric           // emitting the srl/and pair.
35260b57cec5SDimitry Andric           return false;
35270b57cec5SDimitry Andric         }
35280b57cec5SDimitry Andric         Start = CurDAG->getTargetConstant(StartVal, DL, MVT::i32);
35290b57cec5SDimitry Andric       } else {
35300b57cec5SDimitry Andric         // Do not handle the case where the shift amount (can be zero if no srl
35310b57cec5SDimitry Andric         // was found) is not constant. We could handle this case, but it would
35320b57cec5SDimitry Andric         // require run-time logic that would be more expensive than just
35330b57cec5SDimitry Andric         // emitting the srl/and pair.
35340b57cec5SDimitry Andric         return false;
35350b57cec5SDimitry Andric       }
35360b57cec5SDimitry Andric     } else {
35370b57cec5SDimitry Andric       // Do not handle the case where the LHS of the and is not a shift. While
35380b57cec5SDimitry Andric       // it would be trivial to handle this case, it would just transform
35390b57cec5SDimitry Andric       // 'and' -> 'bfe', but 'and' has higher-throughput.
35400b57cec5SDimitry Andric       return false;
35410b57cec5SDimitry Andric     }
35420b57cec5SDimitry Andric   } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
35430b57cec5SDimitry Andric     if (LHS->getOpcode() == ISD::AND) {
35440b57cec5SDimitry Andric       ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
35450b57cec5SDimitry Andric       if (!ShiftCnst) {
35460b57cec5SDimitry Andric         // Shift amount must be constant
35470b57cec5SDimitry Andric         return false;
35480b57cec5SDimitry Andric       }
35490b57cec5SDimitry Andric 
35500b57cec5SDimitry Andric       uint64_t ShiftAmt = ShiftCnst->getZExtValue();
35510b57cec5SDimitry Andric 
35520b57cec5SDimitry Andric       SDValue AndLHS = LHS->getOperand(0);
35530b57cec5SDimitry Andric       SDValue AndRHS = LHS->getOperand(1);
35540b57cec5SDimitry Andric 
35550b57cec5SDimitry Andric       // Canonicalize the AND to have the mask on the RHS
35560b57cec5SDimitry Andric       if (isa<ConstantSDNode>(AndLHS)) {
35570b57cec5SDimitry Andric         std::swap(AndLHS, AndRHS);
35580b57cec5SDimitry Andric       }
35590b57cec5SDimitry Andric 
35600b57cec5SDimitry Andric       ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
35610b57cec5SDimitry Andric       if (!MaskCnst) {
35620b57cec5SDimitry Andric         // Mask must be constant
35630b57cec5SDimitry Andric         return false;
35640b57cec5SDimitry Andric       }
35650b57cec5SDimitry Andric 
35660b57cec5SDimitry Andric       uint64_t MaskVal = MaskCnst->getZExtValue();
35670b57cec5SDimitry Andric       uint64_t NumZeros;
35680b57cec5SDimitry Andric       uint64_t NumBits;
35690b57cec5SDimitry Andric       if (isMask_64(MaskVal)) {
35700b57cec5SDimitry Andric         NumZeros = 0;
35710b57cec5SDimitry Andric         // The number of bits in the result bitfield will be the number of
35720b57cec5SDimitry Andric         // trailing ones (the AND) minus the number of bits we shift off
357306c3fb27SDimitry Andric         NumBits = llvm::countr_one(MaskVal) - ShiftAmt;
35740b57cec5SDimitry Andric       } else if (isShiftedMask_64(MaskVal)) {
357506c3fb27SDimitry Andric         NumZeros = llvm::countr_zero(MaskVal);
357606c3fb27SDimitry Andric         unsigned NumOnes = llvm::countr_one(MaskVal >> NumZeros);
35770b57cec5SDimitry Andric         // The number of bits in the result bitfield will be the number of
35780b57cec5SDimitry Andric         // trailing zeros plus the number of set bits in the mask minus the
35790b57cec5SDimitry Andric         // number of bits we shift off
35800b57cec5SDimitry Andric         NumBits = NumZeros + NumOnes - ShiftAmt;
35810b57cec5SDimitry Andric       } else {
35820b57cec5SDimitry Andric         // This is not a mask we can handle
35830b57cec5SDimitry Andric         return false;
35840b57cec5SDimitry Andric       }
35850b57cec5SDimitry Andric 
35860b57cec5SDimitry Andric       if (ShiftAmt < NumZeros) {
35870b57cec5SDimitry Andric         // Handling this case would require extra logic that would make this
35880b57cec5SDimitry Andric         // transformation non-profitable
35890b57cec5SDimitry Andric         return false;
35900b57cec5SDimitry Andric       }
35910b57cec5SDimitry Andric 
35920b57cec5SDimitry Andric       Val = AndLHS;
35930b57cec5SDimitry Andric       Start = CurDAG->getTargetConstant(ShiftAmt, DL, MVT::i32);
35940b57cec5SDimitry Andric       Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
35950b57cec5SDimitry Andric     } else if (LHS->getOpcode() == ISD::SHL) {
35960b57cec5SDimitry Andric       // Here, we have a pattern like:
35970b57cec5SDimitry Andric       //
35980b57cec5SDimitry Andric       // (sra (shl val, NN), MM)
35990b57cec5SDimitry Andric       // or
36000b57cec5SDimitry Andric       // (srl (shl val, NN), MM)
36010b57cec5SDimitry Andric       //
36020b57cec5SDimitry Andric       // If MM >= NN, we can efficiently optimize this with bfe
36030b57cec5SDimitry Andric       Val = LHS->getOperand(0);
36040b57cec5SDimitry Andric 
36050b57cec5SDimitry Andric       SDValue ShlRHS = LHS->getOperand(1);
36060b57cec5SDimitry Andric       ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
36070b57cec5SDimitry Andric       if (!ShlCnst) {
36080b57cec5SDimitry Andric         // Shift amount must be constant
36090b57cec5SDimitry Andric         return false;
36100b57cec5SDimitry Andric       }
36110b57cec5SDimitry Andric       uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
36120b57cec5SDimitry Andric 
36130b57cec5SDimitry Andric       SDValue ShrRHS = RHS;
36140b57cec5SDimitry Andric       ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
36150b57cec5SDimitry Andric       if (!ShrCnst) {
36160b57cec5SDimitry Andric         // Shift amount must be constant
36170b57cec5SDimitry Andric         return false;
36180b57cec5SDimitry Andric       }
36190b57cec5SDimitry Andric       uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
36200b57cec5SDimitry Andric 
36210b57cec5SDimitry Andric       // To avoid extra codegen and be profitable, we need Outer >= Inner
36220b57cec5SDimitry Andric       if (OuterShiftAmt < InnerShiftAmt) {
36230b57cec5SDimitry Andric         return false;
36240b57cec5SDimitry Andric       }
36250b57cec5SDimitry Andric 
36260b57cec5SDimitry Andric       // If the outer shift is more than the type size, we have no bitfield to
36270b57cec5SDimitry Andric       // extract (since we also check that the inner shift is <= the outer shift
36280b57cec5SDimitry Andric       // then this also implies that the inner shift is < the type size)
36290b57cec5SDimitry Andric       if (OuterShiftAmt >= Val.getValueSizeInBits()) {
36300b57cec5SDimitry Andric         return false;
36310b57cec5SDimitry Andric       }
36320b57cec5SDimitry Andric 
36330b57cec5SDimitry Andric       Start = CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, DL,
36340b57cec5SDimitry Andric                                         MVT::i32);
36350b57cec5SDimitry Andric       Len = CurDAG->getTargetConstant(Val.getValueSizeInBits() - OuterShiftAmt,
36360b57cec5SDimitry Andric                                       DL, MVT::i32);
36370b57cec5SDimitry Andric 
36380b57cec5SDimitry Andric       if (N->getOpcode() == ISD::SRA) {
36390b57cec5SDimitry Andric         // If we have a arithmetic right shift, we need to use the signed bfe
36400b57cec5SDimitry Andric         // variant
36410b57cec5SDimitry Andric         IsSigned = true;
36420b57cec5SDimitry Andric       }
36430b57cec5SDimitry Andric     } else {
36440b57cec5SDimitry Andric       // No can do...
36450b57cec5SDimitry Andric       return false;
36460b57cec5SDimitry Andric     }
36470b57cec5SDimitry Andric   } else {
36480b57cec5SDimitry Andric     // No can do...
36490b57cec5SDimitry Andric     return false;
36500b57cec5SDimitry Andric   }
36510b57cec5SDimitry Andric 
36520b57cec5SDimitry Andric 
36530b57cec5SDimitry Andric   unsigned Opc;
36540b57cec5SDimitry Andric   // For the BFE operations we form here from "and" and "srl", always use the
36550b57cec5SDimitry Andric   // unsigned variants.
36560b57cec5SDimitry Andric   if (Val.getValueType() == MVT::i32) {
36570b57cec5SDimitry Andric     if (IsSigned) {
36580b57cec5SDimitry Andric       Opc = NVPTX::BFE_S32rii;
36590b57cec5SDimitry Andric     } else {
36600b57cec5SDimitry Andric       Opc = NVPTX::BFE_U32rii;
36610b57cec5SDimitry Andric     }
36620b57cec5SDimitry Andric   } else if (Val.getValueType() == MVT::i64) {
36630b57cec5SDimitry Andric     if (IsSigned) {
36640b57cec5SDimitry Andric       Opc = NVPTX::BFE_S64rii;
36650b57cec5SDimitry Andric     } else {
36660b57cec5SDimitry Andric       Opc = NVPTX::BFE_U64rii;
36670b57cec5SDimitry Andric     }
36680b57cec5SDimitry Andric   } else {
36690b57cec5SDimitry Andric     // We cannot handle this type
36700b57cec5SDimitry Andric     return false;
36710b57cec5SDimitry Andric   }
36720b57cec5SDimitry Andric 
36730b57cec5SDimitry Andric   SDValue Ops[] = {
36740b57cec5SDimitry Andric     Val, Start, Len
36750b57cec5SDimitry Andric   };
36760b57cec5SDimitry Andric 
36770b57cec5SDimitry Andric   ReplaceNode(N, CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops));
36780b57cec5SDimitry Andric   return true;
36790b57cec5SDimitry Andric }
36800b57cec5SDimitry Andric 
36810b57cec5SDimitry Andric // SelectDirectAddr - Match a direct address for DAG.
36820b57cec5SDimitry Andric // A direct address could be a globaladdress or externalsymbol.
36830b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
36840b57cec5SDimitry Andric   // Return true if TGA or ES.
36850b57cec5SDimitry Andric   if (N.getOpcode() == ISD::TargetGlobalAddress ||
36860b57cec5SDimitry Andric       N.getOpcode() == ISD::TargetExternalSymbol) {
36870b57cec5SDimitry Andric     Address = N;
36880b57cec5SDimitry Andric     return true;
36890b57cec5SDimitry Andric   }
36900b57cec5SDimitry Andric   if (N.getOpcode() == NVPTXISD::Wrapper) {
36910b57cec5SDimitry Andric     Address = N.getOperand(0);
36920b57cec5SDimitry Andric     return true;
36930b57cec5SDimitry Andric   }
36940b57cec5SDimitry Andric   // addrspacecast(MoveParam(arg_symbol) to addrspace(PARAM)) -> arg_symbol
36950b57cec5SDimitry Andric   if (AddrSpaceCastSDNode *CastN = dyn_cast<AddrSpaceCastSDNode>(N)) {
36960b57cec5SDimitry Andric     if (CastN->getSrcAddressSpace() == ADDRESS_SPACE_GENERIC &&
36970b57cec5SDimitry Andric         CastN->getDestAddressSpace() == ADDRESS_SPACE_PARAM &&
36980b57cec5SDimitry Andric         CastN->getOperand(0).getOpcode() == NVPTXISD::MoveParam)
36990b57cec5SDimitry Andric       return SelectDirectAddr(CastN->getOperand(0).getOperand(0), Address);
37000b57cec5SDimitry Andric   }
37010b57cec5SDimitry Andric   return false;
37020b57cec5SDimitry Andric }
37030b57cec5SDimitry Andric 
37040b57cec5SDimitry Andric // symbol+offset
37050b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
37060b57cec5SDimitry Andric     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
37070b57cec5SDimitry Andric   if (Addr.getOpcode() == ISD::ADD) {
37080b57cec5SDimitry Andric     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
37090b57cec5SDimitry Andric       SDValue base = Addr.getOperand(0);
37100b57cec5SDimitry Andric       if (SelectDirectAddr(base, Base)) {
37110b57cec5SDimitry Andric         Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
37120b57cec5SDimitry Andric                                            mvt);
37130b57cec5SDimitry Andric         return true;
37140b57cec5SDimitry Andric       }
37150b57cec5SDimitry Andric     }
37160b57cec5SDimitry Andric   }
37170b57cec5SDimitry Andric   return false;
37180b57cec5SDimitry Andric }
37190b57cec5SDimitry Andric 
37200b57cec5SDimitry Andric // symbol+offset
37210b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
37220b57cec5SDimitry Andric                                      SDValue &Base, SDValue &Offset) {
37230b57cec5SDimitry Andric   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
37240b57cec5SDimitry Andric }
37250b57cec5SDimitry Andric 
37260b57cec5SDimitry Andric // symbol+offset
37270b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
37280b57cec5SDimitry Andric                                        SDValue &Base, SDValue &Offset) {
37290b57cec5SDimitry Andric   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
37300b57cec5SDimitry Andric }
37310b57cec5SDimitry Andric 
37320b57cec5SDimitry Andric // register+offset
37330b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectADDRri_imp(
37340b57cec5SDimitry Andric     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
37350b57cec5SDimitry Andric   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
37360b57cec5SDimitry Andric     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
37370b57cec5SDimitry Andric     Offset = CurDAG->getTargetConstant(0, SDLoc(OpNode), mvt);
37380b57cec5SDimitry Andric     return true;
37390b57cec5SDimitry Andric   }
37400b57cec5SDimitry Andric   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
37410b57cec5SDimitry Andric       Addr.getOpcode() == ISD::TargetGlobalAddress)
37420b57cec5SDimitry Andric     return false; // direct calls.
37430b57cec5SDimitry Andric 
37440b57cec5SDimitry Andric   if (Addr.getOpcode() == ISD::ADD) {
37450b57cec5SDimitry Andric     if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
37460b57cec5SDimitry Andric       return false;
37470b57cec5SDimitry Andric     }
37480b57cec5SDimitry Andric     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
37490b57cec5SDimitry Andric       if (FrameIndexSDNode *FIN =
37500b57cec5SDimitry Andric               dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
37510b57cec5SDimitry Andric         // Constant offset from frame ref.
37520b57cec5SDimitry Andric         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
37530b57cec5SDimitry Andric       else
37540b57cec5SDimitry Andric         Base = Addr.getOperand(0);
3755*0fca6ea1SDimitry Andric 
3756*0fca6ea1SDimitry Andric       // Offset must fit in a 32-bit signed int in PTX [register+offset] address
3757*0fca6ea1SDimitry Andric       // mode
3758*0fca6ea1SDimitry Andric       if (!CN->getAPIntValue().isSignedIntN(32))
3759*0fca6ea1SDimitry Andric         return false;
3760*0fca6ea1SDimitry Andric 
3761*0fca6ea1SDimitry Andric       Offset = CurDAG->getTargetConstant(CN->getSExtValue(), SDLoc(OpNode),
3762*0fca6ea1SDimitry Andric                                          MVT::i32);
37630b57cec5SDimitry Andric       return true;
37640b57cec5SDimitry Andric     }
37650b57cec5SDimitry Andric   }
37660b57cec5SDimitry Andric   return false;
37670b57cec5SDimitry Andric }
37680b57cec5SDimitry Andric 
37690b57cec5SDimitry Andric // register+offset
37700b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
37710b57cec5SDimitry Andric                                      SDValue &Base, SDValue &Offset) {
37720b57cec5SDimitry Andric   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
37730b57cec5SDimitry Andric }
37740b57cec5SDimitry Andric 
37750b57cec5SDimitry Andric // register+offset
37760b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
37770b57cec5SDimitry Andric                                        SDValue &Base, SDValue &Offset) {
37780b57cec5SDimitry Andric   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
37790b57cec5SDimitry Andric }
37800b57cec5SDimitry Andric 
37810b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
37820b57cec5SDimitry Andric                                                  unsigned int spN) const {
37830b57cec5SDimitry Andric   const Value *Src = nullptr;
37840b57cec5SDimitry Andric   if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
37850b57cec5SDimitry Andric     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
37860b57cec5SDimitry Andric       return true;
37870b57cec5SDimitry Andric     Src = mN->getMemOperand()->getValue();
37880b57cec5SDimitry Andric   }
37890b57cec5SDimitry Andric   if (!Src)
37900b57cec5SDimitry Andric     return false;
37910b57cec5SDimitry Andric   if (auto *PT = dyn_cast<PointerType>(Src->getType()))
37920b57cec5SDimitry Andric     return (PT->getAddressSpace() == spN);
37930b57cec5SDimitry Andric   return false;
37940b57cec5SDimitry Andric }
37950b57cec5SDimitry Andric 
37960b57cec5SDimitry Andric /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
37970b57cec5SDimitry Andric /// inline asm expressions.
37980b57cec5SDimitry Andric bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
37995f757f3fSDimitry Andric     const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
38005f757f3fSDimitry Andric     std::vector<SDValue> &OutOps) {
38010b57cec5SDimitry Andric   SDValue Op0, Op1;
38020b57cec5SDimitry Andric   switch (ConstraintID) {
38030b57cec5SDimitry Andric   default:
38040b57cec5SDimitry Andric     return true;
38055f757f3fSDimitry Andric   case InlineAsm::ConstraintCode::m: // memory
38060b57cec5SDimitry Andric     if (SelectDirectAddr(Op, Op0)) {
38070b57cec5SDimitry Andric       OutOps.push_back(Op0);
38080b57cec5SDimitry Andric       OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
38090b57cec5SDimitry Andric       return false;
38100b57cec5SDimitry Andric     }
38110b57cec5SDimitry Andric     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
38120b57cec5SDimitry Andric       OutOps.push_back(Op0);
38130b57cec5SDimitry Andric       OutOps.push_back(Op1);
38140b57cec5SDimitry Andric       return false;
38150b57cec5SDimitry Andric     }
38160b57cec5SDimitry Andric     break;
38170b57cec5SDimitry Andric   }
38180b57cec5SDimitry Andric   return true;
38190b57cec5SDimitry Andric }
38200b57cec5SDimitry Andric 
3821*0fca6ea1SDimitry Andric void NVPTXDAGToDAGISel::SelectV2I64toI128(SDNode *N) {
3822*0fca6ea1SDimitry Andric   // Lower a CopyToReg with two 64-bit inputs
3823*0fca6ea1SDimitry Andric   // Dst:i128, lo:i64, hi:i64
3824*0fca6ea1SDimitry Andric   //
3825*0fca6ea1SDimitry Andric   // CopyToReg Dst, lo, hi;
3826*0fca6ea1SDimitry Andric   //
3827*0fca6ea1SDimitry Andric   // ==>
3828*0fca6ea1SDimitry Andric   //
3829*0fca6ea1SDimitry Andric   // tmp = V2I64toI128 {lo, hi};
3830*0fca6ea1SDimitry Andric   // CopyToReg Dst, tmp;
3831*0fca6ea1SDimitry Andric   SDValue Dst = N->getOperand(1);
3832*0fca6ea1SDimitry Andric   SDValue Lo = N->getOperand(2);
3833*0fca6ea1SDimitry Andric   SDValue Hi = N->getOperand(3);
3834*0fca6ea1SDimitry Andric 
3835*0fca6ea1SDimitry Andric   SDLoc DL(N);
3836*0fca6ea1SDimitry Andric   SDNode *Mov =
3837*0fca6ea1SDimitry Andric       CurDAG->getMachineNode(NVPTX::V2I64toI128, DL, MVT::i128, {Lo, Hi});
3838*0fca6ea1SDimitry Andric 
3839*0fca6ea1SDimitry Andric   SmallVector<SDValue, 4> NewOps(N->getNumOperands() - 1);
3840*0fca6ea1SDimitry Andric   NewOps[0] = N->getOperand(0);
3841*0fca6ea1SDimitry Andric   NewOps[1] = Dst;
3842*0fca6ea1SDimitry Andric   NewOps[2] = SDValue(Mov, 0);
3843*0fca6ea1SDimitry Andric   if (N->getNumOperands() == 5)
3844*0fca6ea1SDimitry Andric     NewOps[3] = N->getOperand(4);
3845*0fca6ea1SDimitry Andric   SDValue NewValue = CurDAG->getNode(ISD::CopyToReg, DL, SmallVector<EVT>(N->values()), NewOps);
3846*0fca6ea1SDimitry Andric 
3847*0fca6ea1SDimitry Andric   ReplaceNode(N, NewValue.getNode());
3848*0fca6ea1SDimitry Andric }
3849*0fca6ea1SDimitry Andric 
3850*0fca6ea1SDimitry Andric void NVPTXDAGToDAGISel::SelectI128toV2I64(SDNode *N) {
3851*0fca6ea1SDimitry Andric   // Lower CopyFromReg from a 128-bit regs to two 64-bit regs
3852*0fca6ea1SDimitry Andric   // Dst:i128, Src:i128
3853*0fca6ea1SDimitry Andric   //
3854*0fca6ea1SDimitry Andric   // {lo, hi} = CopyFromReg Src
3855*0fca6ea1SDimitry Andric   //
3856*0fca6ea1SDimitry Andric   // ==>
3857*0fca6ea1SDimitry Andric   //
3858*0fca6ea1SDimitry Andric   // {lo, hi} = I128toV2I64 Src
3859*0fca6ea1SDimitry Andric   //
3860*0fca6ea1SDimitry Andric   SDValue Ch = N->getOperand(0);
3861*0fca6ea1SDimitry Andric   SDValue Src = N->getOperand(1);
3862*0fca6ea1SDimitry Andric   SDValue Glue = N->getOperand(2);
3863*0fca6ea1SDimitry Andric   SDLoc DL(N);
3864*0fca6ea1SDimitry Andric 
3865*0fca6ea1SDimitry Andric   // Add Glue and Ch to the operands and results to avoid break the execution
3866*0fca6ea1SDimitry Andric   // order
3867*0fca6ea1SDimitry Andric   SDNode *Mov = CurDAG->getMachineNode(
3868*0fca6ea1SDimitry Andric       NVPTX::I128toV2I64, DL,
3869*0fca6ea1SDimitry Andric       {MVT::i64, MVT::i64, Ch.getValueType(), Glue.getValueType()},
3870*0fca6ea1SDimitry Andric       {Src, Ch, Glue});
3871*0fca6ea1SDimitry Andric 
3872*0fca6ea1SDimitry Andric   ReplaceNode(N, Mov);
3873*0fca6ea1SDimitry Andric }
3874*0fca6ea1SDimitry Andric 
38750b57cec5SDimitry Andric /// GetConvertOpcode - Returns the CVT_ instruction opcode that implements a
38760b57cec5SDimitry Andric /// conversion from \p SrcTy to \p DestTy.
38770b57cec5SDimitry Andric unsigned NVPTXDAGToDAGISel::GetConvertOpcode(MVT DestTy, MVT SrcTy,
387806c3fb27SDimitry Andric                                              LoadSDNode *LdNode) {
387906c3fb27SDimitry Andric   bool IsSigned = LdNode && LdNode->getExtensionType() == ISD::SEXTLOAD;
38800b57cec5SDimitry Andric   switch (SrcTy.SimpleTy) {
38810b57cec5SDimitry Andric   default:
38820b57cec5SDimitry Andric     llvm_unreachable("Unhandled source type");
38830b57cec5SDimitry Andric   case MVT::i8:
38840b57cec5SDimitry Andric     switch (DestTy.SimpleTy) {
38850b57cec5SDimitry Andric     default:
38860b57cec5SDimitry Andric       llvm_unreachable("Unhandled dest type");
38870b57cec5SDimitry Andric     case MVT::i16:
38880b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s16_s8 : NVPTX::CVT_u16_u8;
38890b57cec5SDimitry Andric     case MVT::i32:
38900b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s32_s8 : NVPTX::CVT_u32_u8;
38910b57cec5SDimitry Andric     case MVT::i64:
38920b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s64_s8 : NVPTX::CVT_u64_u8;
38930b57cec5SDimitry Andric     }
38940b57cec5SDimitry Andric   case MVT::i16:
38950b57cec5SDimitry Andric     switch (DestTy.SimpleTy) {
38960b57cec5SDimitry Andric     default:
38970b57cec5SDimitry Andric       llvm_unreachable("Unhandled dest type");
38980b57cec5SDimitry Andric     case MVT::i8:
38990b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s8_s16 : NVPTX::CVT_u8_u16;
39000b57cec5SDimitry Andric     case MVT::i32:
39010b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s32_s16 : NVPTX::CVT_u32_u16;
39020b57cec5SDimitry Andric     case MVT::i64:
39030b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s64_s16 : NVPTX::CVT_u64_u16;
39040b57cec5SDimitry Andric     }
39050b57cec5SDimitry Andric   case MVT::i32:
39060b57cec5SDimitry Andric     switch (DestTy.SimpleTy) {
39070b57cec5SDimitry Andric     default:
39080b57cec5SDimitry Andric       llvm_unreachable("Unhandled dest type");
39090b57cec5SDimitry Andric     case MVT::i8:
39100b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s8_s32 : NVPTX::CVT_u8_u32;
39110b57cec5SDimitry Andric     case MVT::i16:
39120b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s16_s32 : NVPTX::CVT_u16_u32;
39130b57cec5SDimitry Andric     case MVT::i64:
39140b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s64_s32 : NVPTX::CVT_u64_u32;
39150b57cec5SDimitry Andric     }
39160b57cec5SDimitry Andric   case MVT::i64:
39170b57cec5SDimitry Andric     switch (DestTy.SimpleTy) {
39180b57cec5SDimitry Andric     default:
39190b57cec5SDimitry Andric       llvm_unreachable("Unhandled dest type");
39200b57cec5SDimitry Andric     case MVT::i8:
39210b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s8_s64 : NVPTX::CVT_u8_u64;
39220b57cec5SDimitry Andric     case MVT::i16:
39230b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s16_s64 : NVPTX::CVT_u16_u64;
39240b57cec5SDimitry Andric     case MVT::i32:
39250b57cec5SDimitry Andric       return IsSigned ? NVPTX::CVT_s32_s64 : NVPTX::CVT_u32_u64;
39260b57cec5SDimitry Andric     }
392706c3fb27SDimitry Andric   case MVT::f16:
392806c3fb27SDimitry Andric     switch (DestTy.SimpleTy) {
392906c3fb27SDimitry Andric     default:
393006c3fb27SDimitry Andric       llvm_unreachable("Unhandled dest type");
393106c3fb27SDimitry Andric     case MVT::f32:
393206c3fb27SDimitry Andric       return NVPTX::CVT_f32_f16;
393306c3fb27SDimitry Andric     case MVT::f64:
393406c3fb27SDimitry Andric       return NVPTX::CVT_f64_f16;
393506c3fb27SDimitry Andric     }
39360b57cec5SDimitry Andric   }
39370b57cec5SDimitry Andric }
3938