xref: /freebsd-src/contrib/llvm-project/llvm/lib/Target/VE/VVPISelLowering.cpp (revision 81ad626541db97eb356e2c1d4a20eb2a26a766ab)
1*81ad6265SDimitry Andric //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
2*81ad6265SDimitry Andric //
3*81ad6265SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4*81ad6265SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5*81ad6265SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6*81ad6265SDimitry Andric //
7*81ad6265SDimitry Andric //===----------------------------------------------------------------------===//
8*81ad6265SDimitry Andric //
9*81ad6265SDimitry Andric // This file implements the lowering and legalization of vector instructions to
10*81ad6265SDimitry Andric // VVP_*layer SDNodes.
11*81ad6265SDimitry Andric //
12*81ad6265SDimitry Andric //===----------------------------------------------------------------------===//
13*81ad6265SDimitry Andric 
14*81ad6265SDimitry Andric #include "VECustomDAG.h"
15*81ad6265SDimitry Andric #include "VEISelLowering.h"
16*81ad6265SDimitry Andric 
17*81ad6265SDimitry Andric using namespace llvm;
18*81ad6265SDimitry Andric 
19*81ad6265SDimitry Andric #define DEBUG_TYPE "ve-lower"
20*81ad6265SDimitry Andric 
21*81ad6265SDimitry Andric SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
22*81ad6265SDimitry Andric                                               SelectionDAG &DAG) const {
23*81ad6265SDimitry Andric   VECustomDAG CDAG(DAG, Op);
24*81ad6265SDimitry Andric   SDValue AVL =
25*81ad6265SDimitry Andric       CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26*81ad6265SDimitry Andric   SDValue A = Op->getOperand(0);
27*81ad6265SDimitry Andric   SDValue B = Op->getOperand(1);
28*81ad6265SDimitry Andric   SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29*81ad6265SDimitry Andric   SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30*81ad6265SDimitry Andric   SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31*81ad6265SDimitry Andric   SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32*81ad6265SDimitry Andric   unsigned Opc = Op.getOpcode();
33*81ad6265SDimitry Andric   auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34*81ad6265SDimitry Andric   auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35*81ad6265SDimitry Andric   return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36*81ad6265SDimitry Andric }
37*81ad6265SDimitry Andric 
38*81ad6265SDimitry Andric SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
39*81ad6265SDimitry Andric   // Can we represent this as a VVP node.
40*81ad6265SDimitry Andric   const unsigned Opcode = Op->getOpcode();
41*81ad6265SDimitry Andric   auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42*81ad6265SDimitry Andric   if (!VVPOpcodeOpt)
43*81ad6265SDimitry Andric     return SDValue();
44*81ad6265SDimitry Andric   unsigned VVPOpcode = VVPOpcodeOpt.getValue();
45*81ad6265SDimitry Andric   const bool FromVP = ISD::isVPOpcode(Opcode);
46*81ad6265SDimitry Andric 
47*81ad6265SDimitry Andric   // The representative and legalized vector type of this operation.
48*81ad6265SDimitry Andric   VECustomDAG CDAG(DAG, Op);
49*81ad6265SDimitry Andric   // Dispatch to complex lowering functions.
50*81ad6265SDimitry Andric   switch (VVPOpcode) {
51*81ad6265SDimitry Andric   case VEISD::VVP_LOAD:
52*81ad6265SDimitry Andric   case VEISD::VVP_STORE:
53*81ad6265SDimitry Andric     return lowerVVP_LOAD_STORE(Op, CDAG);
54*81ad6265SDimitry Andric   case VEISD::VVP_GATHER:
55*81ad6265SDimitry Andric   case VEISD::VVP_SCATTER:
56*81ad6265SDimitry Andric     return lowerVVP_GATHER_SCATTER(Op, CDAG);
57*81ad6265SDimitry Andric   }
58*81ad6265SDimitry Andric 
59*81ad6265SDimitry Andric   EVT OpVecVT = *getIdiomaticVectorType(Op.getNode());
60*81ad6265SDimitry Andric   EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
61*81ad6265SDimitry Andric   auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
62*81ad6265SDimitry Andric 
63*81ad6265SDimitry Andric   SDValue AVL;
64*81ad6265SDimitry Andric   SDValue Mask;
65*81ad6265SDimitry Andric 
66*81ad6265SDimitry Andric   if (FromVP) {
67*81ad6265SDimitry Andric     // All upstream VP SDNodes always have a mask and avl.
68*81ad6265SDimitry Andric     auto MaskIdx = ISD::getVPMaskIdx(Opcode);
69*81ad6265SDimitry Andric     auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
70*81ad6265SDimitry Andric     if (MaskIdx)
71*81ad6265SDimitry Andric       Mask = Op->getOperand(*MaskIdx);
72*81ad6265SDimitry Andric     if (AVLIdx)
73*81ad6265SDimitry Andric       AVL = Op->getOperand(*AVLIdx);
74*81ad6265SDimitry Andric   }
75*81ad6265SDimitry Andric 
76*81ad6265SDimitry Andric   // Materialize default mask and avl.
77*81ad6265SDimitry Andric   if (!AVL)
78*81ad6265SDimitry Andric     AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
79*81ad6265SDimitry Andric   if (!Mask)
80*81ad6265SDimitry Andric     Mask = CDAG.getConstantMask(Packing, true);
81*81ad6265SDimitry Andric 
82*81ad6265SDimitry Andric   assert(LegalVecVT.isSimple());
83*81ad6265SDimitry Andric   if (isVVPUnaryOp(VVPOpcode))
84*81ad6265SDimitry Andric     return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});
85*81ad6265SDimitry Andric   if (isVVPBinaryOp(VVPOpcode))
86*81ad6265SDimitry Andric     return CDAG.getNode(VVPOpcode, LegalVecVT,
87*81ad6265SDimitry Andric                         {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
88*81ad6265SDimitry Andric   if (isVVPReductionOp(VVPOpcode)) {
89*81ad6265SDimitry Andric     auto SrcHasStart = hasReductionStartParam(Op->getOpcode());
90*81ad6265SDimitry Andric     SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();
91*81ad6265SDimitry Andric     SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
92*81ad6265SDimitry Andric     return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,
93*81ad6265SDimitry Andric                                        VectorV, Mask, AVL, Op->getFlags());
94*81ad6265SDimitry Andric   }
95*81ad6265SDimitry Andric 
96*81ad6265SDimitry Andric   switch (VVPOpcode) {
97*81ad6265SDimitry Andric   default:
98*81ad6265SDimitry Andric     llvm_unreachable("lowerToVVP called for unexpected SDNode.");
99*81ad6265SDimitry Andric   case VEISD::VVP_FFMA: {
100*81ad6265SDimitry Andric     // VE has a swizzled operand order in FMA (compared to LLVM IR and
101*81ad6265SDimitry Andric     // SDNodes).
102*81ad6265SDimitry Andric     auto X = Op->getOperand(2);
103*81ad6265SDimitry Andric     auto Y = Op->getOperand(0);
104*81ad6265SDimitry Andric     auto Z = Op->getOperand(1);
105*81ad6265SDimitry Andric     return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL});
106*81ad6265SDimitry Andric   }
107*81ad6265SDimitry Andric   case VEISD::VVP_SELECT: {
108*81ad6265SDimitry Andric     auto Mask = Op->getOperand(0);
109*81ad6265SDimitry Andric     auto OnTrue = Op->getOperand(1);
110*81ad6265SDimitry Andric     auto OnFalse = Op->getOperand(2);
111*81ad6265SDimitry Andric     return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
112*81ad6265SDimitry Andric   }
113*81ad6265SDimitry Andric   case VEISD::VVP_SETCC: {
114*81ad6265SDimitry Andric     EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
115*81ad6265SDimitry Andric     auto LHS = Op->getOperand(0);
116*81ad6265SDimitry Andric     auto RHS = Op->getOperand(1);
117*81ad6265SDimitry Andric     auto Pred = Op->getOperand(2);
118*81ad6265SDimitry Andric     return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL});
119*81ad6265SDimitry Andric   }
120*81ad6265SDimitry Andric   }
121*81ad6265SDimitry Andric }
122*81ad6265SDimitry Andric 
123*81ad6265SDimitry Andric SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
124*81ad6265SDimitry Andric                                               VECustomDAG &CDAG) const {
125*81ad6265SDimitry Andric   auto VVPOpc = *getVVPOpcode(Op->getOpcode());
126*81ad6265SDimitry Andric   const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
127*81ad6265SDimitry Andric 
128*81ad6265SDimitry Andric   // Shares.
129*81ad6265SDimitry Andric   SDValue BasePtr = getMemoryPtr(Op);
130*81ad6265SDimitry Andric   SDValue Mask = getNodeMask(Op);
131*81ad6265SDimitry Andric   SDValue Chain = getNodeChain(Op);
132*81ad6265SDimitry Andric   SDValue AVL = getNodeAVL(Op);
133*81ad6265SDimitry Andric   // Store specific.
134*81ad6265SDimitry Andric   SDValue Data = getStoredValue(Op);
135*81ad6265SDimitry Andric   // Load specific.
136*81ad6265SDimitry Andric   SDValue PassThru = getNodePassthru(Op);
137*81ad6265SDimitry Andric 
138*81ad6265SDimitry Andric   SDValue StrideV = getLoadStoreStride(Op, CDAG);
139*81ad6265SDimitry Andric 
140*81ad6265SDimitry Andric   auto DataVT = *getIdiomaticVectorType(Op.getNode());
141*81ad6265SDimitry Andric   auto Packing = getTypePacking(DataVT);
142*81ad6265SDimitry Andric 
143*81ad6265SDimitry Andric   // TODO: Infer lower AVL from mask.
144*81ad6265SDimitry Andric   if (!AVL)
145*81ad6265SDimitry Andric     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
146*81ad6265SDimitry Andric 
147*81ad6265SDimitry Andric   // Default to the all-true mask.
148*81ad6265SDimitry Andric   if (!Mask)
149*81ad6265SDimitry Andric     Mask = CDAG.getConstantMask(Packing, true);
150*81ad6265SDimitry Andric 
151*81ad6265SDimitry Andric   if (IsLoad) {
152*81ad6265SDimitry Andric     MVT LegalDataVT = getLegalVectorType(
153*81ad6265SDimitry Andric         Packing, DataVT.getVectorElementType().getSimpleVT());
154*81ad6265SDimitry Andric 
155*81ad6265SDimitry Andric     auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
156*81ad6265SDimitry Andric                                  {Chain, BasePtr, StrideV, Mask, AVL});
157*81ad6265SDimitry Andric 
158*81ad6265SDimitry Andric     if (!PassThru || PassThru->isUndef())
159*81ad6265SDimitry Andric       return NewLoadV;
160*81ad6265SDimitry Andric 
161*81ad6265SDimitry Andric     // Convert passthru to an explicit select node.
162*81ad6265SDimitry Andric     SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
163*81ad6265SDimitry Andric                                  {NewLoadV, PassThru, Mask, AVL});
164*81ad6265SDimitry Andric     SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
165*81ad6265SDimitry Andric 
166*81ad6265SDimitry Andric     // Merge them back into one node.
167*81ad6265SDimitry Andric     return CDAG.getMergeValues({DataV, NewLoadChainV});
168*81ad6265SDimitry Andric   }
169*81ad6265SDimitry Andric 
170*81ad6265SDimitry Andric   // VVP_STORE
171*81ad6265SDimitry Andric   assert(VVPOpc == VEISD::VVP_STORE);
172*81ad6265SDimitry Andric   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
173*81ad6265SDimitry Andric                       {Chain, Data, BasePtr, StrideV, Mask, AVL});
174*81ad6265SDimitry Andric }
175*81ad6265SDimitry Andric 
176*81ad6265SDimitry Andric SDValue VETargetLowering::splitPackedLoadStore(SDValue Op,
177*81ad6265SDimitry Andric                                                VECustomDAG &CDAG) const {
178*81ad6265SDimitry Andric   auto VVPOC = *getVVPOpcode(Op.getOpcode());
179*81ad6265SDimitry Andric   assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
180*81ad6265SDimitry Andric 
181*81ad6265SDimitry Andric   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
182*81ad6265SDimitry Andric   assert(getTypePacking(DataVT) == Packing::Dense &&
183*81ad6265SDimitry Andric          "Can only split packed load/store");
184*81ad6265SDimitry Andric   MVT SplitDataVT = splitVectorType(DataVT);
185*81ad6265SDimitry Andric 
186*81ad6265SDimitry Andric   assert(!getNodePassthru(Op) &&
187*81ad6265SDimitry Andric          "Should have been folded in lowering to VVP layer");
188*81ad6265SDimitry Andric 
189*81ad6265SDimitry Andric   // Analyze the operation
190*81ad6265SDimitry Andric   SDValue PackedMask = getNodeMask(Op);
191*81ad6265SDimitry Andric   SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
192*81ad6265SDimitry Andric   SDValue PackPtr = getMemoryPtr(Op);
193*81ad6265SDimitry Andric   SDValue PackData = getStoredValue(Op);
194*81ad6265SDimitry Andric   SDValue PackStride = getLoadStoreStride(Op, CDAG);
195*81ad6265SDimitry Andric 
196*81ad6265SDimitry Andric   unsigned ChainResIdx = PackData ? 0 : 1;
197*81ad6265SDimitry Andric 
198*81ad6265SDimitry Andric   SDValue PartOps[2];
199*81ad6265SDimitry Andric 
200*81ad6265SDimitry Andric   SDValue UpperPartAVL; // we will use this for packing things back together
201*81ad6265SDimitry Andric   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
202*81ad6265SDimitry Andric     // VP ops already have an explicit mask and AVL. When expanding from non-VP
203*81ad6265SDimitry Andric     // attach those additional inputs here.
204*81ad6265SDimitry Andric     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
205*81ad6265SDimitry Andric 
206*81ad6265SDimitry Andric     // Keep track of the (higher) lvl.
207*81ad6265SDimitry Andric     if (Part == PackElem::Hi)
208*81ad6265SDimitry Andric       UpperPartAVL = SplitTM.AVL;
209*81ad6265SDimitry Andric 
210*81ad6265SDimitry Andric     // Attach non-predicating value operands
211*81ad6265SDimitry Andric     SmallVector<SDValue, 4> OpVec;
212*81ad6265SDimitry Andric 
213*81ad6265SDimitry Andric     // Chain
214*81ad6265SDimitry Andric     OpVec.push_back(getNodeChain(Op));
215*81ad6265SDimitry Andric 
216*81ad6265SDimitry Andric     // Data
217*81ad6265SDimitry Andric     if (PackData) {
218*81ad6265SDimitry Andric       SDValue PartData =
219*81ad6265SDimitry Andric           CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
220*81ad6265SDimitry Andric       OpVec.push_back(PartData);
221*81ad6265SDimitry Andric     }
222*81ad6265SDimitry Andric 
223*81ad6265SDimitry Andric     // Ptr & Stride
224*81ad6265SDimitry Andric     // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
225*81ad6265SDimitry Andric     // Stride info
226*81ad6265SDimitry Andric     // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
227*81ad6265SDimitry Andric     OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
228*81ad6265SDimitry Andric     OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
229*81ad6265SDimitry Andric 
230*81ad6265SDimitry Andric     // Add predicating args and generate part node
231*81ad6265SDimitry Andric     OpVec.push_back(SplitTM.Mask);
232*81ad6265SDimitry Andric     OpVec.push_back(SplitTM.AVL);
233*81ad6265SDimitry Andric 
234*81ad6265SDimitry Andric     if (PackData) {
235*81ad6265SDimitry Andric       // Store
236*81ad6265SDimitry Andric       PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
237*81ad6265SDimitry Andric     } else {
238*81ad6265SDimitry Andric       // Load
239*81ad6265SDimitry Andric       PartOps[(int)Part] =
240*81ad6265SDimitry Andric           CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
241*81ad6265SDimitry Andric     }
242*81ad6265SDimitry Andric   }
243*81ad6265SDimitry Andric 
244*81ad6265SDimitry Andric   // Merge the chains
245*81ad6265SDimitry Andric   SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
246*81ad6265SDimitry Andric   SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
247*81ad6265SDimitry Andric   SDValue FusedChains =
248*81ad6265SDimitry Andric       CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
249*81ad6265SDimitry Andric 
250*81ad6265SDimitry Andric   // Chain only [store]
251*81ad6265SDimitry Andric   if (PackData)
252*81ad6265SDimitry Andric     return FusedChains;
253*81ad6265SDimitry Andric 
254*81ad6265SDimitry Andric   // Re-pack into full packed vector result
255*81ad6265SDimitry Andric   MVT PackedVT =
256*81ad6265SDimitry Andric       getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
257*81ad6265SDimitry Andric   SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
258*81ad6265SDimitry Andric                                     PartOps[(int)PackElem::Hi], UpperPartAVL);
259*81ad6265SDimitry Andric 
260*81ad6265SDimitry Andric   return CDAG.getMergeValues({PackedVals, FusedChains});
261*81ad6265SDimitry Andric }
262*81ad6265SDimitry Andric 
263*81ad6265SDimitry Andric SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op,
264*81ad6265SDimitry Andric                                                   VECustomDAG &CDAG) const {
265*81ad6265SDimitry Andric   EVT DataVT = *getIdiomaticVectorType(Op.getNode());
266*81ad6265SDimitry Andric   auto Packing = getTypePacking(DataVT);
267*81ad6265SDimitry Andric   MVT LegalDataVT =
268*81ad6265SDimitry Andric       getLegalVectorType(Packing, DataVT.getVectorElementType().getSimpleVT());
269*81ad6265SDimitry Andric 
270*81ad6265SDimitry Andric   SDValue AVL = getAnnotatedNodeAVL(Op).first;
271*81ad6265SDimitry Andric   SDValue Index = getGatherScatterIndex(Op);
272*81ad6265SDimitry Andric   SDValue BasePtr = getMemoryPtr(Op);
273*81ad6265SDimitry Andric   SDValue Mask = getNodeMask(Op);
274*81ad6265SDimitry Andric   SDValue Chain = getNodeChain(Op);
275*81ad6265SDimitry Andric   SDValue Scale = getGatherScatterScale(Op);
276*81ad6265SDimitry Andric   SDValue PassThru = getNodePassthru(Op);
277*81ad6265SDimitry Andric   SDValue StoredValue = getStoredValue(Op);
278*81ad6265SDimitry Andric   if (PassThru && PassThru->isUndef())
279*81ad6265SDimitry Andric     PassThru = SDValue();
280*81ad6265SDimitry Andric 
281*81ad6265SDimitry Andric   bool IsScatter = (bool)StoredValue;
282*81ad6265SDimitry Andric 
283*81ad6265SDimitry Andric   // TODO: Infer lower AVL from mask.
284*81ad6265SDimitry Andric   if (!AVL)
285*81ad6265SDimitry Andric     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
286*81ad6265SDimitry Andric 
287*81ad6265SDimitry Andric   // Default to the all-true mask.
288*81ad6265SDimitry Andric   if (!Mask)
289*81ad6265SDimitry Andric     Mask = CDAG.getConstantMask(Packing, true);
290*81ad6265SDimitry Andric 
291*81ad6265SDimitry Andric   SDValue AddressVec =
292*81ad6265SDimitry Andric       CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL);
293*81ad6265SDimitry Andric   if (IsScatter)
294*81ad6265SDimitry Andric     return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other,
295*81ad6265SDimitry Andric                         {Chain, StoredValue, AddressVec, Mask, AVL});
296*81ad6265SDimitry Andric 
297*81ad6265SDimitry Andric   // Gather.
298*81ad6265SDimitry Andric   SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other},
299*81ad6265SDimitry Andric                                   {Chain, AddressVec, Mask, AVL});
300*81ad6265SDimitry Andric 
301*81ad6265SDimitry Andric   if (!PassThru)
302*81ad6265SDimitry Andric     return NewLoadV;
303*81ad6265SDimitry Andric 
304*81ad6265SDimitry Andric   // TODO: Use vvp_select
305*81ad6265SDimitry Andric   SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT,
306*81ad6265SDimitry Andric                                {NewLoadV, PassThru, Mask, AVL});
307*81ad6265SDimitry Andric   SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
308*81ad6265SDimitry Andric   return CDAG.getMergeValues({DataV, NewLoadChainV});
309*81ad6265SDimitry Andric }
310*81ad6265SDimitry Andric 
311*81ad6265SDimitry Andric SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,
312*81ad6265SDimitry Andric                                                       VECustomDAG &CDAG) const {
313*81ad6265SDimitry Andric   LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
314*81ad6265SDimitry Andric   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
315*81ad6265SDimitry Andric 
316*81ad6265SDimitry Andric   // TODO: Recognize packable load,store.
317*81ad6265SDimitry Andric   if (isPackedVectorType(DataVT))
318*81ad6265SDimitry Andric     return splitPackedLoadStore(Op, CDAG);
319*81ad6265SDimitry Andric 
320*81ad6265SDimitry Andric   return legalizePackedAVL(Op, CDAG);
321*81ad6265SDimitry Andric }
322*81ad6265SDimitry Andric 
323*81ad6265SDimitry Andric SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
324*81ad6265SDimitry Andric                                                    SelectionDAG &DAG) const {
325*81ad6265SDimitry Andric   LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
326*81ad6265SDimitry Andric   VECustomDAG CDAG(DAG, Op);
327*81ad6265SDimitry Andric 
328*81ad6265SDimitry Andric   // Dispatch to specialized legalization functions.
329*81ad6265SDimitry Andric   switch (Op->getOpcode()) {
330*81ad6265SDimitry Andric   case VEISD::VVP_LOAD:
331*81ad6265SDimitry Andric   case VEISD::VVP_STORE:
332*81ad6265SDimitry Andric     return legalizeInternalLoadStoreOp(Op, CDAG);
333*81ad6265SDimitry Andric   }
334*81ad6265SDimitry Andric 
335*81ad6265SDimitry Andric   EVT IdiomVT = Op.getValueType();
336*81ad6265SDimitry Andric   if (isPackedVectorType(IdiomVT) &&
337*81ad6265SDimitry Andric       !supportsPackedMode(Op.getOpcode(), IdiomVT))
338*81ad6265SDimitry Andric     return splitVectorOp(Op, CDAG);
339*81ad6265SDimitry Andric 
340*81ad6265SDimitry Andric   // TODO: Implement odd/even splitting.
341*81ad6265SDimitry Andric   return legalizePackedAVL(Op, CDAG);
342*81ad6265SDimitry Andric }
343*81ad6265SDimitry Andric 
344*81ad6265SDimitry Andric SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
345*81ad6265SDimitry Andric   MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
346*81ad6265SDimitry Andric 
347*81ad6265SDimitry Andric   auto AVLPos = getAVLPos(Op->getOpcode());
348*81ad6265SDimitry Andric   auto MaskPos = getMaskPos(Op->getOpcode());
349*81ad6265SDimitry Andric 
350*81ad6265SDimitry Andric   SDValue PackedMask = getNodeMask(Op);
351*81ad6265SDimitry Andric   auto AVLPair = getAnnotatedNodeAVL(Op);
352*81ad6265SDimitry Andric   SDValue PackedAVL = AVLPair.first;
353*81ad6265SDimitry Andric   assert(!AVLPair.second && "Expecting non pack-legalized oepration");
354*81ad6265SDimitry Andric 
355*81ad6265SDimitry Andric   // request the parts
356*81ad6265SDimitry Andric   SDValue PartOps[2];
357*81ad6265SDimitry Andric 
358*81ad6265SDimitry Andric   SDValue UpperPartAVL; // we will use this for packing things back together
359*81ad6265SDimitry Andric   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
360*81ad6265SDimitry Andric     // VP ops already have an explicit mask and AVL. When expanding from non-VP
361*81ad6265SDimitry Andric     // attach those additional inputs here.
362*81ad6265SDimitry Andric     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
363*81ad6265SDimitry Andric 
364*81ad6265SDimitry Andric     if (Part == PackElem::Hi)
365*81ad6265SDimitry Andric       UpperPartAVL = SplitTM.AVL;
366*81ad6265SDimitry Andric 
367*81ad6265SDimitry Andric     // Attach non-predicating value operands
368*81ad6265SDimitry Andric     SmallVector<SDValue, 4> OpVec;
369*81ad6265SDimitry Andric     for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
370*81ad6265SDimitry Andric       if (AVLPos && ((int)i) == *AVLPos)
371*81ad6265SDimitry Andric         continue;
372*81ad6265SDimitry Andric       if (MaskPos && ((int)i) == *MaskPos)
373*81ad6265SDimitry Andric         continue;
374*81ad6265SDimitry Andric 
375*81ad6265SDimitry Andric       // Value operand
376*81ad6265SDimitry Andric       auto PackedOperand = Op.getOperand(i);
377*81ad6265SDimitry Andric       auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
378*81ad6265SDimitry Andric       SDValue PartV =
379*81ad6265SDimitry Andric           CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
380*81ad6265SDimitry Andric       OpVec.push_back(PartV);
381*81ad6265SDimitry Andric     }
382*81ad6265SDimitry Andric 
383*81ad6265SDimitry Andric     // Add predicating args and generate part node.
384*81ad6265SDimitry Andric     OpVec.push_back(SplitTM.Mask);
385*81ad6265SDimitry Andric     OpVec.push_back(SplitTM.AVL);
386*81ad6265SDimitry Andric     // Emit legal VVP nodes.
387*81ad6265SDimitry Andric     PartOps[(int)Part] =
388*81ad6265SDimitry Andric         CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
389*81ad6265SDimitry Andric   }
390*81ad6265SDimitry Andric 
391*81ad6265SDimitry Andric   // Re-package vectors.
392*81ad6265SDimitry Andric   return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
393*81ad6265SDimitry Andric                       PartOps[(int)PackElem::Hi], UpperPartAVL);
394*81ad6265SDimitry Andric }
395*81ad6265SDimitry Andric 
396*81ad6265SDimitry Andric SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
397*81ad6265SDimitry Andric                                             VECustomDAG &CDAG) const {
398*81ad6265SDimitry Andric   LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
399*81ad6265SDimitry Andric   // Only required for VEC and VVP ops.
400*81ad6265SDimitry Andric   if (!isVVPOrVEC(Op->getOpcode()))
401*81ad6265SDimitry Andric     return Op;
402*81ad6265SDimitry Andric 
403*81ad6265SDimitry Andric   // Operation already has a legal AVL.
404*81ad6265SDimitry Andric   auto AVL = getNodeAVL(Op);
405*81ad6265SDimitry Andric   if (isLegalAVL(AVL))
406*81ad6265SDimitry Andric     return Op;
407*81ad6265SDimitry Andric 
408*81ad6265SDimitry Andric   // Half and round up EVL for 32bit element types.
409*81ad6265SDimitry Andric   SDValue LegalAVL = AVL;
410*81ad6265SDimitry Andric   MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
411*81ad6265SDimitry Andric   if (isPackedVectorType(IdiomVT)) {
412*81ad6265SDimitry Andric     assert(maySafelyIgnoreMask(Op) &&
413*81ad6265SDimitry Andric            "TODO Shift predication from EVL into Mask");
414*81ad6265SDimitry Andric 
415*81ad6265SDimitry Andric     if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
416*81ad6265SDimitry Andric       LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
417*81ad6265SDimitry Andric     } else {
418*81ad6265SDimitry Andric       auto ConstOne = CDAG.getConstant(1, MVT::i32);
419*81ad6265SDimitry Andric       auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
420*81ad6265SDimitry Andric       LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
421*81ad6265SDimitry Andric     }
422*81ad6265SDimitry Andric   }
423*81ad6265SDimitry Andric 
424*81ad6265SDimitry Andric   SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
425*81ad6265SDimitry Andric 
426*81ad6265SDimitry Andric   // Copy the operand list.
427*81ad6265SDimitry Andric   int NumOp = Op->getNumOperands();
428*81ad6265SDimitry Andric   auto AVLPos = getAVLPos(Op->getOpcode());
429*81ad6265SDimitry Andric   std::vector<SDValue> FixedOperands;
430*81ad6265SDimitry Andric   for (int i = 0; i < NumOp; ++i) {
431*81ad6265SDimitry Andric     if (AVLPos && (i == *AVLPos)) {
432*81ad6265SDimitry Andric       FixedOperands.push_back(AnnotatedLegalAVL);
433*81ad6265SDimitry Andric       continue;
434*81ad6265SDimitry Andric     }
435*81ad6265SDimitry Andric     FixedOperands.push_back(Op->getOperand(i));
436*81ad6265SDimitry Andric   }
437*81ad6265SDimitry Andric 
438*81ad6265SDimitry Andric   // Clone the operation with fixed operands.
439*81ad6265SDimitry Andric   auto Flags = Op->getFlags();
440*81ad6265SDimitry Andric   SDValue NewN =
441*81ad6265SDimitry Andric       CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
442*81ad6265SDimitry Andric   return NewN;
443*81ad6265SDimitry Andric }
444