1*81ad6265SDimitry Andric //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===// 2*81ad6265SDimitry Andric // 3*81ad6265SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4*81ad6265SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5*81ad6265SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6*81ad6265SDimitry Andric // 7*81ad6265SDimitry Andric //===----------------------------------------------------------------------===// 8*81ad6265SDimitry Andric // 9*81ad6265SDimitry Andric // This file implements the lowering and legalization of vector instructions to 10*81ad6265SDimitry Andric // VVP_*layer SDNodes. 11*81ad6265SDimitry Andric // 12*81ad6265SDimitry Andric //===----------------------------------------------------------------------===// 13*81ad6265SDimitry Andric 14*81ad6265SDimitry Andric #include "VECustomDAG.h" 15*81ad6265SDimitry Andric #include "VEISelLowering.h" 16*81ad6265SDimitry Andric 17*81ad6265SDimitry Andric using namespace llvm; 18*81ad6265SDimitry Andric 19*81ad6265SDimitry Andric #define DEBUG_TYPE "ve-lower" 20*81ad6265SDimitry Andric 21*81ad6265SDimitry Andric SDValue VETargetLowering::splitMaskArithmetic(SDValue Op, 22*81ad6265SDimitry Andric SelectionDAG &DAG) const { 23*81ad6265SDimitry Andric VECustomDAG CDAG(DAG, Op); 24*81ad6265SDimitry Andric SDValue AVL = 25*81ad6265SDimitry Andric CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32); 26*81ad6265SDimitry Andric SDValue A = Op->getOperand(0); 27*81ad6265SDimitry Andric SDValue B = Op->getOperand(1); 28*81ad6265SDimitry Andric SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL); 29*81ad6265SDimitry Andric SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL); 30*81ad6265SDimitry Andric SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL); 31*81ad6265SDimitry Andric SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL); 32*81ad6265SDimitry Andric unsigned Opc = Op.getOpcode(); 33*81ad6265SDimitry Andric auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB}); 34*81ad6265SDimitry Andric auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB}); 35*81ad6265SDimitry Andric return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL); 36*81ad6265SDimitry Andric } 37*81ad6265SDimitry Andric 38*81ad6265SDimitry Andric SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const { 39*81ad6265SDimitry Andric // Can we represent this as a VVP node. 40*81ad6265SDimitry Andric const unsigned Opcode = Op->getOpcode(); 41*81ad6265SDimitry Andric auto VVPOpcodeOpt = getVVPOpcode(Opcode); 42*81ad6265SDimitry Andric if (!VVPOpcodeOpt) 43*81ad6265SDimitry Andric return SDValue(); 44*81ad6265SDimitry Andric unsigned VVPOpcode = VVPOpcodeOpt.getValue(); 45*81ad6265SDimitry Andric const bool FromVP = ISD::isVPOpcode(Opcode); 46*81ad6265SDimitry Andric 47*81ad6265SDimitry Andric // The representative and legalized vector type of this operation. 48*81ad6265SDimitry Andric VECustomDAG CDAG(DAG, Op); 49*81ad6265SDimitry Andric // Dispatch to complex lowering functions. 50*81ad6265SDimitry Andric switch (VVPOpcode) { 51*81ad6265SDimitry Andric case VEISD::VVP_LOAD: 52*81ad6265SDimitry Andric case VEISD::VVP_STORE: 53*81ad6265SDimitry Andric return lowerVVP_LOAD_STORE(Op, CDAG); 54*81ad6265SDimitry Andric case VEISD::VVP_GATHER: 55*81ad6265SDimitry Andric case VEISD::VVP_SCATTER: 56*81ad6265SDimitry Andric return lowerVVP_GATHER_SCATTER(Op, CDAG); 57*81ad6265SDimitry Andric } 58*81ad6265SDimitry Andric 59*81ad6265SDimitry Andric EVT OpVecVT = *getIdiomaticVectorType(Op.getNode()); 60*81ad6265SDimitry Andric EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT); 61*81ad6265SDimitry Andric auto Packing = getTypePacking(LegalVecVT.getSimpleVT()); 62*81ad6265SDimitry Andric 63*81ad6265SDimitry Andric SDValue AVL; 64*81ad6265SDimitry Andric SDValue Mask; 65*81ad6265SDimitry Andric 66*81ad6265SDimitry Andric if (FromVP) { 67*81ad6265SDimitry Andric // All upstream VP SDNodes always have a mask and avl. 68*81ad6265SDimitry Andric auto MaskIdx = ISD::getVPMaskIdx(Opcode); 69*81ad6265SDimitry Andric auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode); 70*81ad6265SDimitry Andric if (MaskIdx) 71*81ad6265SDimitry Andric Mask = Op->getOperand(*MaskIdx); 72*81ad6265SDimitry Andric if (AVLIdx) 73*81ad6265SDimitry Andric AVL = Op->getOperand(*AVLIdx); 74*81ad6265SDimitry Andric } 75*81ad6265SDimitry Andric 76*81ad6265SDimitry Andric // Materialize default mask and avl. 77*81ad6265SDimitry Andric if (!AVL) 78*81ad6265SDimitry Andric AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32); 79*81ad6265SDimitry Andric if (!Mask) 80*81ad6265SDimitry Andric Mask = CDAG.getConstantMask(Packing, true); 81*81ad6265SDimitry Andric 82*81ad6265SDimitry Andric assert(LegalVecVT.isSimple()); 83*81ad6265SDimitry Andric if (isVVPUnaryOp(VVPOpcode)) 84*81ad6265SDimitry Andric return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL}); 85*81ad6265SDimitry Andric if (isVVPBinaryOp(VVPOpcode)) 86*81ad6265SDimitry Andric return CDAG.getNode(VVPOpcode, LegalVecVT, 87*81ad6265SDimitry Andric {Op->getOperand(0), Op->getOperand(1), Mask, AVL}); 88*81ad6265SDimitry Andric if (isVVPReductionOp(VVPOpcode)) { 89*81ad6265SDimitry Andric auto SrcHasStart = hasReductionStartParam(Op->getOpcode()); 90*81ad6265SDimitry Andric SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue(); 91*81ad6265SDimitry Andric SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0); 92*81ad6265SDimitry Andric return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV, 93*81ad6265SDimitry Andric VectorV, Mask, AVL, Op->getFlags()); 94*81ad6265SDimitry Andric } 95*81ad6265SDimitry Andric 96*81ad6265SDimitry Andric switch (VVPOpcode) { 97*81ad6265SDimitry Andric default: 98*81ad6265SDimitry Andric llvm_unreachable("lowerToVVP called for unexpected SDNode."); 99*81ad6265SDimitry Andric case VEISD::VVP_FFMA: { 100*81ad6265SDimitry Andric // VE has a swizzled operand order in FMA (compared to LLVM IR and 101*81ad6265SDimitry Andric // SDNodes). 102*81ad6265SDimitry Andric auto X = Op->getOperand(2); 103*81ad6265SDimitry Andric auto Y = Op->getOperand(0); 104*81ad6265SDimitry Andric auto Z = Op->getOperand(1); 105*81ad6265SDimitry Andric return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL}); 106*81ad6265SDimitry Andric } 107*81ad6265SDimitry Andric case VEISD::VVP_SELECT: { 108*81ad6265SDimitry Andric auto Mask = Op->getOperand(0); 109*81ad6265SDimitry Andric auto OnTrue = Op->getOperand(1); 110*81ad6265SDimitry Andric auto OnFalse = Op->getOperand(2); 111*81ad6265SDimitry Andric return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL}); 112*81ad6265SDimitry Andric } 113*81ad6265SDimitry Andric case VEISD::VVP_SETCC: { 114*81ad6265SDimitry Andric EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 115*81ad6265SDimitry Andric auto LHS = Op->getOperand(0); 116*81ad6265SDimitry Andric auto RHS = Op->getOperand(1); 117*81ad6265SDimitry Andric auto Pred = Op->getOperand(2); 118*81ad6265SDimitry Andric return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL}); 119*81ad6265SDimitry Andric } 120*81ad6265SDimitry Andric } 121*81ad6265SDimitry Andric } 122*81ad6265SDimitry Andric 123*81ad6265SDimitry Andric SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op, 124*81ad6265SDimitry Andric VECustomDAG &CDAG) const { 125*81ad6265SDimitry Andric auto VVPOpc = *getVVPOpcode(Op->getOpcode()); 126*81ad6265SDimitry Andric const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD); 127*81ad6265SDimitry Andric 128*81ad6265SDimitry Andric // Shares. 129*81ad6265SDimitry Andric SDValue BasePtr = getMemoryPtr(Op); 130*81ad6265SDimitry Andric SDValue Mask = getNodeMask(Op); 131*81ad6265SDimitry Andric SDValue Chain = getNodeChain(Op); 132*81ad6265SDimitry Andric SDValue AVL = getNodeAVL(Op); 133*81ad6265SDimitry Andric // Store specific. 134*81ad6265SDimitry Andric SDValue Data = getStoredValue(Op); 135*81ad6265SDimitry Andric // Load specific. 136*81ad6265SDimitry Andric SDValue PassThru = getNodePassthru(Op); 137*81ad6265SDimitry Andric 138*81ad6265SDimitry Andric SDValue StrideV = getLoadStoreStride(Op, CDAG); 139*81ad6265SDimitry Andric 140*81ad6265SDimitry Andric auto DataVT = *getIdiomaticVectorType(Op.getNode()); 141*81ad6265SDimitry Andric auto Packing = getTypePacking(DataVT); 142*81ad6265SDimitry Andric 143*81ad6265SDimitry Andric // TODO: Infer lower AVL from mask. 144*81ad6265SDimitry Andric if (!AVL) 145*81ad6265SDimitry Andric AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32); 146*81ad6265SDimitry Andric 147*81ad6265SDimitry Andric // Default to the all-true mask. 148*81ad6265SDimitry Andric if (!Mask) 149*81ad6265SDimitry Andric Mask = CDAG.getConstantMask(Packing, true); 150*81ad6265SDimitry Andric 151*81ad6265SDimitry Andric if (IsLoad) { 152*81ad6265SDimitry Andric MVT LegalDataVT = getLegalVectorType( 153*81ad6265SDimitry Andric Packing, DataVT.getVectorElementType().getSimpleVT()); 154*81ad6265SDimitry Andric 155*81ad6265SDimitry Andric auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other}, 156*81ad6265SDimitry Andric {Chain, BasePtr, StrideV, Mask, AVL}); 157*81ad6265SDimitry Andric 158*81ad6265SDimitry Andric if (!PassThru || PassThru->isUndef()) 159*81ad6265SDimitry Andric return NewLoadV; 160*81ad6265SDimitry Andric 161*81ad6265SDimitry Andric // Convert passthru to an explicit select node. 162*81ad6265SDimitry Andric SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT, 163*81ad6265SDimitry Andric {NewLoadV, PassThru, Mask, AVL}); 164*81ad6265SDimitry Andric SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1); 165*81ad6265SDimitry Andric 166*81ad6265SDimitry Andric // Merge them back into one node. 167*81ad6265SDimitry Andric return CDAG.getMergeValues({DataV, NewLoadChainV}); 168*81ad6265SDimitry Andric } 169*81ad6265SDimitry Andric 170*81ad6265SDimitry Andric // VVP_STORE 171*81ad6265SDimitry Andric assert(VVPOpc == VEISD::VVP_STORE); 172*81ad6265SDimitry Andric return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(), 173*81ad6265SDimitry Andric {Chain, Data, BasePtr, StrideV, Mask, AVL}); 174*81ad6265SDimitry Andric } 175*81ad6265SDimitry Andric 176*81ad6265SDimitry Andric SDValue VETargetLowering::splitPackedLoadStore(SDValue Op, 177*81ad6265SDimitry Andric VECustomDAG &CDAG) const { 178*81ad6265SDimitry Andric auto VVPOC = *getVVPOpcode(Op.getOpcode()); 179*81ad6265SDimitry Andric assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE)); 180*81ad6265SDimitry Andric 181*81ad6265SDimitry Andric MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT(); 182*81ad6265SDimitry Andric assert(getTypePacking(DataVT) == Packing::Dense && 183*81ad6265SDimitry Andric "Can only split packed load/store"); 184*81ad6265SDimitry Andric MVT SplitDataVT = splitVectorType(DataVT); 185*81ad6265SDimitry Andric 186*81ad6265SDimitry Andric assert(!getNodePassthru(Op) && 187*81ad6265SDimitry Andric "Should have been folded in lowering to VVP layer"); 188*81ad6265SDimitry Andric 189*81ad6265SDimitry Andric // Analyze the operation 190*81ad6265SDimitry Andric SDValue PackedMask = getNodeMask(Op); 191*81ad6265SDimitry Andric SDValue PackedAVL = getAnnotatedNodeAVL(Op).first; 192*81ad6265SDimitry Andric SDValue PackPtr = getMemoryPtr(Op); 193*81ad6265SDimitry Andric SDValue PackData = getStoredValue(Op); 194*81ad6265SDimitry Andric SDValue PackStride = getLoadStoreStride(Op, CDAG); 195*81ad6265SDimitry Andric 196*81ad6265SDimitry Andric unsigned ChainResIdx = PackData ? 0 : 1; 197*81ad6265SDimitry Andric 198*81ad6265SDimitry Andric SDValue PartOps[2]; 199*81ad6265SDimitry Andric 200*81ad6265SDimitry Andric SDValue UpperPartAVL; // we will use this for packing things back together 201*81ad6265SDimitry Andric for (PackElem Part : {PackElem::Hi, PackElem::Lo}) { 202*81ad6265SDimitry Andric // VP ops already have an explicit mask and AVL. When expanding from non-VP 203*81ad6265SDimitry Andric // attach those additional inputs here. 204*81ad6265SDimitry Andric auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part); 205*81ad6265SDimitry Andric 206*81ad6265SDimitry Andric // Keep track of the (higher) lvl. 207*81ad6265SDimitry Andric if (Part == PackElem::Hi) 208*81ad6265SDimitry Andric UpperPartAVL = SplitTM.AVL; 209*81ad6265SDimitry Andric 210*81ad6265SDimitry Andric // Attach non-predicating value operands 211*81ad6265SDimitry Andric SmallVector<SDValue, 4> OpVec; 212*81ad6265SDimitry Andric 213*81ad6265SDimitry Andric // Chain 214*81ad6265SDimitry Andric OpVec.push_back(getNodeChain(Op)); 215*81ad6265SDimitry Andric 216*81ad6265SDimitry Andric // Data 217*81ad6265SDimitry Andric if (PackData) { 218*81ad6265SDimitry Andric SDValue PartData = 219*81ad6265SDimitry Andric CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL); 220*81ad6265SDimitry Andric OpVec.push_back(PartData); 221*81ad6265SDimitry Andric } 222*81ad6265SDimitry Andric 223*81ad6265SDimitry Andric // Ptr & Stride 224*81ad6265SDimitry Andric // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes) 225*81ad6265SDimitry Andric // Stride info 226*81ad6265SDimitry Andric // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode); 227*81ad6265SDimitry Andric OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part)); 228*81ad6265SDimitry Andric OpVec.push_back(CDAG.getSplitPtrStride(PackStride)); 229*81ad6265SDimitry Andric 230*81ad6265SDimitry Andric // Add predicating args and generate part node 231*81ad6265SDimitry Andric OpVec.push_back(SplitTM.Mask); 232*81ad6265SDimitry Andric OpVec.push_back(SplitTM.AVL); 233*81ad6265SDimitry Andric 234*81ad6265SDimitry Andric if (PackData) { 235*81ad6265SDimitry Andric // Store 236*81ad6265SDimitry Andric PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec); 237*81ad6265SDimitry Andric } else { 238*81ad6265SDimitry Andric // Load 239*81ad6265SDimitry Andric PartOps[(int)Part] = 240*81ad6265SDimitry Andric CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec); 241*81ad6265SDimitry Andric } 242*81ad6265SDimitry Andric } 243*81ad6265SDimitry Andric 244*81ad6265SDimitry Andric // Merge the chains 245*81ad6265SDimitry Andric SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx); 246*81ad6265SDimitry Andric SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx); 247*81ad6265SDimitry Andric SDValue FusedChains = 248*81ad6265SDimitry Andric CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain}); 249*81ad6265SDimitry Andric 250*81ad6265SDimitry Andric // Chain only [store] 251*81ad6265SDimitry Andric if (PackData) 252*81ad6265SDimitry Andric return FusedChains; 253*81ad6265SDimitry Andric 254*81ad6265SDimitry Andric // Re-pack into full packed vector result 255*81ad6265SDimitry Andric MVT PackedVT = 256*81ad6265SDimitry Andric getLegalVectorType(Packing::Dense, DataVT.getVectorElementType()); 257*81ad6265SDimitry Andric SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo], 258*81ad6265SDimitry Andric PartOps[(int)PackElem::Hi], UpperPartAVL); 259*81ad6265SDimitry Andric 260*81ad6265SDimitry Andric return CDAG.getMergeValues({PackedVals, FusedChains}); 261*81ad6265SDimitry Andric } 262*81ad6265SDimitry Andric 263*81ad6265SDimitry Andric SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op, 264*81ad6265SDimitry Andric VECustomDAG &CDAG) const { 265*81ad6265SDimitry Andric EVT DataVT = *getIdiomaticVectorType(Op.getNode()); 266*81ad6265SDimitry Andric auto Packing = getTypePacking(DataVT); 267*81ad6265SDimitry Andric MVT LegalDataVT = 268*81ad6265SDimitry Andric getLegalVectorType(Packing, DataVT.getVectorElementType().getSimpleVT()); 269*81ad6265SDimitry Andric 270*81ad6265SDimitry Andric SDValue AVL = getAnnotatedNodeAVL(Op).first; 271*81ad6265SDimitry Andric SDValue Index = getGatherScatterIndex(Op); 272*81ad6265SDimitry Andric SDValue BasePtr = getMemoryPtr(Op); 273*81ad6265SDimitry Andric SDValue Mask = getNodeMask(Op); 274*81ad6265SDimitry Andric SDValue Chain = getNodeChain(Op); 275*81ad6265SDimitry Andric SDValue Scale = getGatherScatterScale(Op); 276*81ad6265SDimitry Andric SDValue PassThru = getNodePassthru(Op); 277*81ad6265SDimitry Andric SDValue StoredValue = getStoredValue(Op); 278*81ad6265SDimitry Andric if (PassThru && PassThru->isUndef()) 279*81ad6265SDimitry Andric PassThru = SDValue(); 280*81ad6265SDimitry Andric 281*81ad6265SDimitry Andric bool IsScatter = (bool)StoredValue; 282*81ad6265SDimitry Andric 283*81ad6265SDimitry Andric // TODO: Infer lower AVL from mask. 284*81ad6265SDimitry Andric if (!AVL) 285*81ad6265SDimitry Andric AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32); 286*81ad6265SDimitry Andric 287*81ad6265SDimitry Andric // Default to the all-true mask. 288*81ad6265SDimitry Andric if (!Mask) 289*81ad6265SDimitry Andric Mask = CDAG.getConstantMask(Packing, true); 290*81ad6265SDimitry Andric 291*81ad6265SDimitry Andric SDValue AddressVec = 292*81ad6265SDimitry Andric CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL); 293*81ad6265SDimitry Andric if (IsScatter) 294*81ad6265SDimitry Andric return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other, 295*81ad6265SDimitry Andric {Chain, StoredValue, AddressVec, Mask, AVL}); 296*81ad6265SDimitry Andric 297*81ad6265SDimitry Andric // Gather. 298*81ad6265SDimitry Andric SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other}, 299*81ad6265SDimitry Andric {Chain, AddressVec, Mask, AVL}); 300*81ad6265SDimitry Andric 301*81ad6265SDimitry Andric if (!PassThru) 302*81ad6265SDimitry Andric return NewLoadV; 303*81ad6265SDimitry Andric 304*81ad6265SDimitry Andric // TODO: Use vvp_select 305*81ad6265SDimitry Andric SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT, 306*81ad6265SDimitry Andric {NewLoadV, PassThru, Mask, AVL}); 307*81ad6265SDimitry Andric SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1); 308*81ad6265SDimitry Andric return CDAG.getMergeValues({DataV, NewLoadChainV}); 309*81ad6265SDimitry Andric } 310*81ad6265SDimitry Andric 311*81ad6265SDimitry Andric SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op, 312*81ad6265SDimitry Andric VECustomDAG &CDAG) const { 313*81ad6265SDimitry Andric LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";); 314*81ad6265SDimitry Andric MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT(); 315*81ad6265SDimitry Andric 316*81ad6265SDimitry Andric // TODO: Recognize packable load,store. 317*81ad6265SDimitry Andric if (isPackedVectorType(DataVT)) 318*81ad6265SDimitry Andric return splitPackedLoadStore(Op, CDAG); 319*81ad6265SDimitry Andric 320*81ad6265SDimitry Andric return legalizePackedAVL(Op, CDAG); 321*81ad6265SDimitry Andric } 322*81ad6265SDimitry Andric 323*81ad6265SDimitry Andric SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op, 324*81ad6265SDimitry Andric SelectionDAG &DAG) const { 325*81ad6265SDimitry Andric LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";); 326*81ad6265SDimitry Andric VECustomDAG CDAG(DAG, Op); 327*81ad6265SDimitry Andric 328*81ad6265SDimitry Andric // Dispatch to specialized legalization functions. 329*81ad6265SDimitry Andric switch (Op->getOpcode()) { 330*81ad6265SDimitry Andric case VEISD::VVP_LOAD: 331*81ad6265SDimitry Andric case VEISD::VVP_STORE: 332*81ad6265SDimitry Andric return legalizeInternalLoadStoreOp(Op, CDAG); 333*81ad6265SDimitry Andric } 334*81ad6265SDimitry Andric 335*81ad6265SDimitry Andric EVT IdiomVT = Op.getValueType(); 336*81ad6265SDimitry Andric if (isPackedVectorType(IdiomVT) && 337*81ad6265SDimitry Andric !supportsPackedMode(Op.getOpcode(), IdiomVT)) 338*81ad6265SDimitry Andric return splitVectorOp(Op, CDAG); 339*81ad6265SDimitry Andric 340*81ad6265SDimitry Andric // TODO: Implement odd/even splitting. 341*81ad6265SDimitry Andric return legalizePackedAVL(Op, CDAG); 342*81ad6265SDimitry Andric } 343*81ad6265SDimitry Andric 344*81ad6265SDimitry Andric SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const { 345*81ad6265SDimitry Andric MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType()); 346*81ad6265SDimitry Andric 347*81ad6265SDimitry Andric auto AVLPos = getAVLPos(Op->getOpcode()); 348*81ad6265SDimitry Andric auto MaskPos = getMaskPos(Op->getOpcode()); 349*81ad6265SDimitry Andric 350*81ad6265SDimitry Andric SDValue PackedMask = getNodeMask(Op); 351*81ad6265SDimitry Andric auto AVLPair = getAnnotatedNodeAVL(Op); 352*81ad6265SDimitry Andric SDValue PackedAVL = AVLPair.first; 353*81ad6265SDimitry Andric assert(!AVLPair.second && "Expecting non pack-legalized oepration"); 354*81ad6265SDimitry Andric 355*81ad6265SDimitry Andric // request the parts 356*81ad6265SDimitry Andric SDValue PartOps[2]; 357*81ad6265SDimitry Andric 358*81ad6265SDimitry Andric SDValue UpperPartAVL; // we will use this for packing things back together 359*81ad6265SDimitry Andric for (PackElem Part : {PackElem::Hi, PackElem::Lo}) { 360*81ad6265SDimitry Andric // VP ops already have an explicit mask and AVL. When expanding from non-VP 361*81ad6265SDimitry Andric // attach those additional inputs here. 362*81ad6265SDimitry Andric auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part); 363*81ad6265SDimitry Andric 364*81ad6265SDimitry Andric if (Part == PackElem::Hi) 365*81ad6265SDimitry Andric UpperPartAVL = SplitTM.AVL; 366*81ad6265SDimitry Andric 367*81ad6265SDimitry Andric // Attach non-predicating value operands 368*81ad6265SDimitry Andric SmallVector<SDValue, 4> OpVec; 369*81ad6265SDimitry Andric for (unsigned i = 0; i < Op.getNumOperands(); ++i) { 370*81ad6265SDimitry Andric if (AVLPos && ((int)i) == *AVLPos) 371*81ad6265SDimitry Andric continue; 372*81ad6265SDimitry Andric if (MaskPos && ((int)i) == *MaskPos) 373*81ad6265SDimitry Andric continue; 374*81ad6265SDimitry Andric 375*81ad6265SDimitry Andric // Value operand 376*81ad6265SDimitry Andric auto PackedOperand = Op.getOperand(i); 377*81ad6265SDimitry Andric auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType()); 378*81ad6265SDimitry Andric SDValue PartV = 379*81ad6265SDimitry Andric CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL); 380*81ad6265SDimitry Andric OpVec.push_back(PartV); 381*81ad6265SDimitry Andric } 382*81ad6265SDimitry Andric 383*81ad6265SDimitry Andric // Add predicating args and generate part node. 384*81ad6265SDimitry Andric OpVec.push_back(SplitTM.Mask); 385*81ad6265SDimitry Andric OpVec.push_back(SplitTM.AVL); 386*81ad6265SDimitry Andric // Emit legal VVP nodes. 387*81ad6265SDimitry Andric PartOps[(int)Part] = 388*81ad6265SDimitry Andric CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags()); 389*81ad6265SDimitry Andric } 390*81ad6265SDimitry Andric 391*81ad6265SDimitry Andric // Re-package vectors. 392*81ad6265SDimitry Andric return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo], 393*81ad6265SDimitry Andric PartOps[(int)PackElem::Hi], UpperPartAVL); 394*81ad6265SDimitry Andric } 395*81ad6265SDimitry Andric 396*81ad6265SDimitry Andric SDValue VETargetLowering::legalizePackedAVL(SDValue Op, 397*81ad6265SDimitry Andric VECustomDAG &CDAG) const { 398*81ad6265SDimitry Andric LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";); 399*81ad6265SDimitry Andric // Only required for VEC and VVP ops. 400*81ad6265SDimitry Andric if (!isVVPOrVEC(Op->getOpcode())) 401*81ad6265SDimitry Andric return Op; 402*81ad6265SDimitry Andric 403*81ad6265SDimitry Andric // Operation already has a legal AVL. 404*81ad6265SDimitry Andric auto AVL = getNodeAVL(Op); 405*81ad6265SDimitry Andric if (isLegalAVL(AVL)) 406*81ad6265SDimitry Andric return Op; 407*81ad6265SDimitry Andric 408*81ad6265SDimitry Andric // Half and round up EVL for 32bit element types. 409*81ad6265SDimitry Andric SDValue LegalAVL = AVL; 410*81ad6265SDimitry Andric MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT(); 411*81ad6265SDimitry Andric if (isPackedVectorType(IdiomVT)) { 412*81ad6265SDimitry Andric assert(maySafelyIgnoreMask(Op) && 413*81ad6265SDimitry Andric "TODO Shift predication from EVL into Mask"); 414*81ad6265SDimitry Andric 415*81ad6265SDimitry Andric if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) { 416*81ad6265SDimitry Andric LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32); 417*81ad6265SDimitry Andric } else { 418*81ad6265SDimitry Andric auto ConstOne = CDAG.getConstant(1, MVT::i32); 419*81ad6265SDimitry Andric auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne}); 420*81ad6265SDimitry Andric LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne}); 421*81ad6265SDimitry Andric } 422*81ad6265SDimitry Andric } 423*81ad6265SDimitry Andric 424*81ad6265SDimitry Andric SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL); 425*81ad6265SDimitry Andric 426*81ad6265SDimitry Andric // Copy the operand list. 427*81ad6265SDimitry Andric int NumOp = Op->getNumOperands(); 428*81ad6265SDimitry Andric auto AVLPos = getAVLPos(Op->getOpcode()); 429*81ad6265SDimitry Andric std::vector<SDValue> FixedOperands; 430*81ad6265SDimitry Andric for (int i = 0; i < NumOp; ++i) { 431*81ad6265SDimitry Andric if (AVLPos && (i == *AVLPos)) { 432*81ad6265SDimitry Andric FixedOperands.push_back(AnnotatedLegalAVL); 433*81ad6265SDimitry Andric continue; 434*81ad6265SDimitry Andric } 435*81ad6265SDimitry Andric FixedOperands.push_back(Op->getOperand(i)); 436*81ad6265SDimitry Andric } 437*81ad6265SDimitry Andric 438*81ad6265SDimitry Andric // Clone the operation with fixed operands. 439*81ad6265SDimitry Andric auto Flags = Op->getFlags(); 440*81ad6265SDimitry Andric SDValue NewN = 441*81ad6265SDimitry Andric CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags); 442*81ad6265SDimitry Andric return NewN; 443*81ad6265SDimitry Andric } 444