xref: /llvm-project/llvm/lib/Target/VE/VVPISelLowering.cpp (revision 2e2395651e4327eec820424f9d647b856745aeb4)
17d926b71SSimon Moll //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
27d926b71SSimon Moll //
37d926b71SSimon Moll // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
47d926b71SSimon Moll // See https://llvm.org/LICENSE.txt for license information.
57d926b71SSimon Moll // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
67d926b71SSimon Moll //
77d926b71SSimon Moll //===----------------------------------------------------------------------===//
87d926b71SSimon Moll //
97d926b71SSimon Moll // This file implements the lowering and legalization of vector instructions to
107d926b71SSimon Moll // VVP_*layer SDNodes.
117d926b71SSimon Moll //
127d926b71SSimon Moll //===----------------------------------------------------------------------===//
137d926b71SSimon Moll 
147d926b71SSimon Moll #include "VECustomDAG.h"
157d926b71SSimon Moll #include "VEISelLowering.h"
167d926b71SSimon Moll 
177d926b71SSimon Moll using namespace llvm;
187d926b71SSimon Moll 
197d926b71SSimon Moll #define DEBUG_TYPE "ve-lower"
207d926b71SSimon Moll 
splitMaskArithmetic(SDValue Op,SelectionDAG & DAG) const21606320edSSimon Moll SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
22606320edSSimon Moll                                               SelectionDAG &DAG) const {
23606320edSSimon Moll   VECustomDAG CDAG(DAG, Op);
24606320edSSimon Moll   SDValue AVL =
25606320edSSimon Moll       CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26606320edSSimon Moll   SDValue A = Op->getOperand(0);
27606320edSSimon Moll   SDValue B = Op->getOperand(1);
28606320edSSimon Moll   SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29606320edSSimon Moll   SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30606320edSSimon Moll   SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31606320edSSimon Moll   SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32606320edSSimon Moll   unsigned Opc = Op.getOpcode();
33606320edSSimon Moll   auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34606320edSSimon Moll   auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35606320edSSimon Moll   return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36606320edSSimon Moll }
37606320edSSimon Moll 
lowerToVVP(SDValue Op,SelectionDAG & DAG) const38606320edSSimon Moll SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
39606320edSSimon Moll   // Can we represent this as a VVP node.
40606320edSSimon Moll   const unsigned Opcode = Op->getOpcode();
41606320edSSimon Moll   auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42a7938c74SKazu Hirata   if (!VVPOpcodeOpt)
43606320edSSimon Moll     return SDValue();
4421c4dc79SFangrui Song   unsigned VVPOpcode = *VVPOpcodeOpt;
45606320edSSimon Moll   const bool FromVP = ISD::isVPOpcode(Opcode);
46606320edSSimon Moll 
47606320edSSimon Moll   // The representative and legalized vector type of this operation.
48606320edSSimon Moll   VECustomDAG CDAG(DAG, Op);
499ebaec46SSimon Moll   // Dispatch to complex lowering functions.
509ebaec46SSimon Moll   switch (VVPOpcode) {
519ebaec46SSimon Moll   case VEISD::VVP_LOAD:
529ebaec46SSimon Moll   case VEISD::VVP_STORE:
539ebaec46SSimon Moll     return lowerVVP_LOAD_STORE(Op, CDAG);
54a5f12623SSimon Moll   case VEISD::VVP_GATHER:
55a5f12623SSimon Moll   case VEISD::VVP_SCATTER:
56a5f12623SSimon Moll     return lowerVVP_GATHER_SCATTER(Op, CDAG);
57a5f12623SSimon Moll   }
589ebaec46SSimon Moll 
59f318d1e2SSimon Moll   EVT OpVecVT = *getIdiomaticVectorType(Op.getNode());
60606320edSSimon Moll   EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
61606320edSSimon Moll   auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
62606320edSSimon Moll 
63606320edSSimon Moll   SDValue AVL;
64606320edSSimon Moll   SDValue Mask;
65606320edSSimon Moll 
66606320edSSimon Moll   if (FromVP) {
67606320edSSimon Moll     // All upstream VP SDNodes always have a mask and avl.
68606320edSSimon Moll     auto MaskIdx = ISD::getVPMaskIdx(Opcode);
69606320edSSimon Moll     auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
70606320edSSimon Moll     if (MaskIdx)
71606320edSSimon Moll       Mask = Op->getOperand(*MaskIdx);
72606320edSSimon Moll     if (AVLIdx)
73606320edSSimon Moll       AVL = Op->getOperand(*AVLIdx);
74606320edSSimon Moll   }
75606320edSSimon Moll 
76606320edSSimon Moll   // Materialize default mask and avl.
77606320edSSimon Moll   if (!AVL)
78606320edSSimon Moll     AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
79606320edSSimon Moll   if (!Mask)
80606320edSSimon Moll     Mask = CDAG.getConstantMask(Packing, true);
81606320edSSimon Moll 
82606320edSSimon Moll   assert(LegalVecVT.isSimple());
8391fad116SSimon Moll   if (isVVPUnaryOp(VVPOpcode))
8491fad116SSimon Moll     return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});
8591fad116SSimon Moll   if (isVVPBinaryOp(VVPOpcode))
86606320edSSimon Moll     return CDAG.getNode(VVPOpcode, LegalVecVT,
87606320edSSimon Moll                         {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
88f318d1e2SSimon Moll   if (isVVPReductionOp(VVPOpcode)) {
89f318d1e2SSimon Moll     auto SrcHasStart = hasReductionStartParam(Op->getOpcode());
90f318d1e2SSimon Moll     SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();
91f318d1e2SSimon Moll     SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
92f318d1e2SSimon Moll     return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,
93f318d1e2SSimon Moll                                        VectorV, Mask, AVL, Op->getFlags());
94f318d1e2SSimon Moll   }
95f318d1e2SSimon Moll 
963297571eSSimon Moll   switch (VVPOpcode) {
973297571eSSimon Moll   default:
983297571eSSimon Moll     llvm_unreachable("lowerToVVP called for unexpected SDNode.");
993297571eSSimon Moll   case VEISD::VVP_FFMA: {
1003297571eSSimon Moll     // VE has a swizzled operand order in FMA (compared to LLVM IR and
1013297571eSSimon Moll     // SDNodes).
1023297571eSSimon Moll     auto X = Op->getOperand(2);
1033297571eSSimon Moll     auto Y = Op->getOperand(0);
1043297571eSSimon Moll     auto Z = Op->getOperand(1);
1053297571eSSimon Moll     return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL});
1063297571eSSimon Moll   }
1073297571eSSimon Moll   case VEISD::VVP_SELECT: {
108606320edSSimon Moll     auto Mask = Op->getOperand(0);
109606320edSSimon Moll     auto OnTrue = Op->getOperand(1);
110606320edSSimon Moll     auto OnFalse = Op->getOperand(2);
111606320edSSimon Moll     return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
112606320edSSimon Moll   }
1133297571eSSimon Moll   case VEISD::VVP_SETCC: {
114f318d1e2SSimon Moll     EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
115606320edSSimon Moll     auto LHS = Op->getOperand(0);
116606320edSSimon Moll     auto RHS = Op->getOperand(1);
117606320edSSimon Moll     auto Pred = Op->getOperand(2);
118f318d1e2SSimon Moll     return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL});
119606320edSSimon Moll   }
1203297571eSSimon Moll   }
121606320edSSimon Moll }
122606320edSSimon Moll 
lowerVVP_LOAD_STORE(SDValue Op,VECustomDAG & CDAG) const1239ebaec46SSimon Moll SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
1249ebaec46SSimon Moll                                               VECustomDAG &CDAG) const {
1259ebaec46SSimon Moll   auto VVPOpc = *getVVPOpcode(Op->getOpcode());
1269ebaec46SSimon Moll   const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
1279ebaec46SSimon Moll 
1289ebaec46SSimon Moll   // Shares.
1299ebaec46SSimon Moll   SDValue BasePtr = getMemoryPtr(Op);
1309ebaec46SSimon Moll   SDValue Mask = getNodeMask(Op);
1319ebaec46SSimon Moll   SDValue Chain = getNodeChain(Op);
1329ebaec46SSimon Moll   SDValue AVL = getNodeAVL(Op);
1339ebaec46SSimon Moll   // Store specific.
1349ebaec46SSimon Moll   SDValue Data = getStoredValue(Op);
1359ebaec46SSimon Moll   // Load specific.
1369ebaec46SSimon Moll   SDValue PassThru = getNodePassthru(Op);
1379ebaec46SSimon Moll 
1386ac3d8efSSimon Moll   SDValue StrideV = getLoadStoreStride(Op, CDAG);
1396ac3d8efSSimon Moll 
1409ebaec46SSimon Moll   auto DataVT = *getIdiomaticVectorType(Op.getNode());
1419ebaec46SSimon Moll   auto Packing = getTypePacking(DataVT);
1429ebaec46SSimon Moll 
1439ebaec46SSimon Moll   // TODO: Infer lower AVL from mask.
1449ebaec46SSimon Moll   if (!AVL)
1459ebaec46SSimon Moll     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
1469ebaec46SSimon Moll 
1479ebaec46SSimon Moll   // Default to the all-true mask.
1489ebaec46SSimon Moll   if (!Mask)
1499ebaec46SSimon Moll     Mask = CDAG.getConstantMask(Packing, true);
1509ebaec46SSimon Moll 
1519ebaec46SSimon Moll   if (IsLoad) {
1529ebaec46SSimon Moll     MVT LegalDataVT = getLegalVectorType(
1539ebaec46SSimon Moll         Packing, DataVT.getVectorElementType().getSimpleVT());
1549ebaec46SSimon Moll 
1559ebaec46SSimon Moll     auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
1569ebaec46SSimon Moll                                  {Chain, BasePtr, StrideV, Mask, AVL});
1579ebaec46SSimon Moll 
1589ebaec46SSimon Moll     if (!PassThru || PassThru->isUndef())
1599ebaec46SSimon Moll       return NewLoadV;
1609ebaec46SSimon Moll 
1619ebaec46SSimon Moll     // Convert passthru to an explicit select node.
1629ebaec46SSimon Moll     SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
1639ebaec46SSimon Moll                                  {NewLoadV, PassThru, Mask, AVL});
1649ebaec46SSimon Moll     SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
1659ebaec46SSimon Moll 
1669ebaec46SSimon Moll     // Merge them back into one node.
1679ebaec46SSimon Moll     return CDAG.getMergeValues({DataV, NewLoadChainV});
1689ebaec46SSimon Moll   }
1699ebaec46SSimon Moll 
1709ebaec46SSimon Moll   // VVP_STORE
1719ebaec46SSimon Moll   assert(VVPOpc == VEISD::VVP_STORE);
172*2e239565SKazushi (Jam) Marukawa   if (getTypeAction(*CDAG.getDAG()->getContext(), Data.getValueType()) !=
173*2e239565SKazushi (Jam) Marukawa       TargetLowering::TypeLegal)
174*2e239565SKazushi (Jam) Marukawa     // Doesn't lower store instruction if an operand is not lowered yet.
175*2e239565SKazushi (Jam) Marukawa     // If it isn't, return SDValue().  In this way, LLVM will try to lower
176*2e239565SKazushi (Jam) Marukawa     // store instruction again after lowering all operands.
177*2e239565SKazushi (Jam) Marukawa     return SDValue();
1789ebaec46SSimon Moll   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
1799ebaec46SSimon Moll                       {Chain, Data, BasePtr, StrideV, Mask, AVL});
1809ebaec46SSimon Moll }
1819ebaec46SSimon Moll 
splitPackedLoadStore(SDValue Op,VECustomDAG & CDAG) const182c574c54eSSimon Moll SDValue VETargetLowering::splitPackedLoadStore(SDValue Op,
183c574c54eSSimon Moll                                                VECustomDAG &CDAG) const {
184c574c54eSSimon Moll   auto VVPOC = *getVVPOpcode(Op.getOpcode());
185c574c54eSSimon Moll   assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
186c574c54eSSimon Moll 
187c574c54eSSimon Moll   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
188c574c54eSSimon Moll   assert(getTypePacking(DataVT) == Packing::Dense &&
189c574c54eSSimon Moll          "Can only split packed load/store");
190c574c54eSSimon Moll   MVT SplitDataVT = splitVectorType(DataVT);
191c574c54eSSimon Moll 
1926cf1bd3aSKazu Hirata   assert(!getNodePassthru(Op) &&
1936cf1bd3aSKazu Hirata          "Should have been folded in lowering to VVP layer");
194c574c54eSSimon Moll 
195c574c54eSSimon Moll   // Analyze the operation
196c574c54eSSimon Moll   SDValue PackedMask = getNodeMask(Op);
197c574c54eSSimon Moll   SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
198c574c54eSSimon Moll   SDValue PackPtr = getMemoryPtr(Op);
199c574c54eSSimon Moll   SDValue PackData = getStoredValue(Op);
200c574c54eSSimon Moll   SDValue PackStride = getLoadStoreStride(Op, CDAG);
201c574c54eSSimon Moll 
202c574c54eSSimon Moll   unsigned ChainResIdx = PackData ? 0 : 1;
203c574c54eSSimon Moll 
204c574c54eSSimon Moll   SDValue PartOps[2];
205c574c54eSSimon Moll 
206c574c54eSSimon Moll   SDValue UpperPartAVL; // we will use this for packing things back together
207c574c54eSSimon Moll   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
208c574c54eSSimon Moll     // VP ops already have an explicit mask and AVL. When expanding from non-VP
209c574c54eSSimon Moll     // attach those additional inputs here.
210c574c54eSSimon Moll     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
211c574c54eSSimon Moll 
212c574c54eSSimon Moll     // Keep track of the (higher) lvl.
213c574c54eSSimon Moll     if (Part == PackElem::Hi)
214c574c54eSSimon Moll       UpperPartAVL = SplitTM.AVL;
215c574c54eSSimon Moll 
216c574c54eSSimon Moll     // Attach non-predicating value operands
217c574c54eSSimon Moll     SmallVector<SDValue, 4> OpVec;
218c574c54eSSimon Moll 
219c574c54eSSimon Moll     // Chain
220c574c54eSSimon Moll     OpVec.push_back(getNodeChain(Op));
221c574c54eSSimon Moll 
222c574c54eSSimon Moll     // Data
223c574c54eSSimon Moll     if (PackData) {
224c574c54eSSimon Moll       SDValue PartData =
225c574c54eSSimon Moll           CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
226c574c54eSSimon Moll       OpVec.push_back(PartData);
227c574c54eSSimon Moll     }
228c574c54eSSimon Moll 
229c574c54eSSimon Moll     // Ptr & Stride
230c574c54eSSimon Moll     // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
231c574c54eSSimon Moll     // Stride info
232c574c54eSSimon Moll     // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
233c574c54eSSimon Moll     OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
234c574c54eSSimon Moll     OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
235c574c54eSSimon Moll 
236c574c54eSSimon Moll     // Add predicating args and generate part node
237c574c54eSSimon Moll     OpVec.push_back(SplitTM.Mask);
238c574c54eSSimon Moll     OpVec.push_back(SplitTM.AVL);
239c574c54eSSimon Moll 
240c574c54eSSimon Moll     if (PackData) {
241c574c54eSSimon Moll       // Store
242c574c54eSSimon Moll       PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
243c574c54eSSimon Moll     } else {
244c574c54eSSimon Moll       // Load
245c574c54eSSimon Moll       PartOps[(int)Part] =
246c574c54eSSimon Moll           CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
247c574c54eSSimon Moll     }
248c574c54eSSimon Moll   }
249c574c54eSSimon Moll 
250c574c54eSSimon Moll   // Merge the chains
251c574c54eSSimon Moll   SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
252c574c54eSSimon Moll   SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
253c574c54eSSimon Moll   SDValue FusedChains =
254c574c54eSSimon Moll       CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
255c574c54eSSimon Moll 
256c574c54eSSimon Moll   // Chain only [store]
257c574c54eSSimon Moll   if (PackData)
258c574c54eSSimon Moll     return FusedChains;
259c574c54eSSimon Moll 
260c574c54eSSimon Moll   // Re-pack into full packed vector result
261c574c54eSSimon Moll   MVT PackedVT =
262c574c54eSSimon Moll       getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
263c574c54eSSimon Moll   SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
264c574c54eSSimon Moll                                     PartOps[(int)PackElem::Hi], UpperPartAVL);
265c574c54eSSimon Moll 
266c574c54eSSimon Moll   return CDAG.getMergeValues({PackedVals, FusedChains});
267c574c54eSSimon Moll }
268c574c54eSSimon Moll 
lowerVVP_GATHER_SCATTER(SDValue Op,VECustomDAG & CDAG) const269a5f12623SSimon Moll SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op,
270a5f12623SSimon Moll                                                   VECustomDAG &CDAG) const {
271a5f12623SSimon Moll   EVT DataVT = *getIdiomaticVectorType(Op.getNode());
272a5f12623SSimon Moll   auto Packing = getTypePacking(DataVT);
273a5f12623SSimon Moll   MVT LegalDataVT =
274a5f12623SSimon Moll       getLegalVectorType(Packing, DataVT.getVectorElementType().getSimpleVT());
275a5f12623SSimon Moll 
276a5f12623SSimon Moll   SDValue AVL = getAnnotatedNodeAVL(Op).first;
277a5f12623SSimon Moll   SDValue Index = getGatherScatterIndex(Op);
278a5f12623SSimon Moll   SDValue BasePtr = getMemoryPtr(Op);
279a5f12623SSimon Moll   SDValue Mask = getNodeMask(Op);
280a5f12623SSimon Moll   SDValue Chain = getNodeChain(Op);
281a5f12623SSimon Moll   SDValue Scale = getGatherScatterScale(Op);
282a5f12623SSimon Moll   SDValue PassThru = getNodePassthru(Op);
283a5f12623SSimon Moll   SDValue StoredValue = getStoredValue(Op);
284a5f12623SSimon Moll   if (PassThru && PassThru->isUndef())
285a5f12623SSimon Moll     PassThru = SDValue();
286a5f12623SSimon Moll 
287a5f12623SSimon Moll   bool IsScatter = (bool)StoredValue;
288a5f12623SSimon Moll 
289a5f12623SSimon Moll   // TODO: Infer lower AVL from mask.
290a5f12623SSimon Moll   if (!AVL)
291a5f12623SSimon Moll     AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
292a5f12623SSimon Moll 
293a5f12623SSimon Moll   // Default to the all-true mask.
294a5f12623SSimon Moll   if (!Mask)
295a5f12623SSimon Moll     Mask = CDAG.getConstantMask(Packing, true);
296a5f12623SSimon Moll 
297a5f12623SSimon Moll   SDValue AddressVec =
298a5f12623SSimon Moll       CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL);
299a5f12623SSimon Moll   if (IsScatter)
300a5f12623SSimon Moll     return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other,
301a5f12623SSimon Moll                         {Chain, StoredValue, AddressVec, Mask, AVL});
302a5f12623SSimon Moll 
303a5f12623SSimon Moll   // Gather.
304a5f12623SSimon Moll   SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other},
305a5f12623SSimon Moll                                   {Chain, AddressVec, Mask, AVL});
306a5f12623SSimon Moll 
307a5f12623SSimon Moll   if (!PassThru)
308a5f12623SSimon Moll     return NewLoadV;
309a5f12623SSimon Moll 
310a5f12623SSimon Moll   // TODO: Use vvp_select
311a5f12623SSimon Moll   SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT,
312a5f12623SSimon Moll                                {NewLoadV, PassThru, Mask, AVL});
313a5f12623SSimon Moll   SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
314a5f12623SSimon Moll   return CDAG.getMergeValues({DataV, NewLoadChainV});
315a5f12623SSimon Moll }
316a5f12623SSimon Moll 
legalizeInternalLoadStoreOp(SDValue Op,VECustomDAG & CDAG) const317c574c54eSSimon Moll SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,
318c574c54eSSimon Moll                                                       VECustomDAG &CDAG) const {
319c574c54eSSimon Moll   LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
320c574c54eSSimon Moll   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
321c574c54eSSimon Moll 
322c574c54eSSimon Moll   // TODO: Recognize packable load,store.
323c574c54eSSimon Moll   if (isPackedVectorType(DataVT))
324c574c54eSSimon Moll     return splitPackedLoadStore(Op, CDAG);
325c574c54eSSimon Moll 
326c574c54eSSimon Moll   return legalizePackedAVL(Op, CDAG);
327c574c54eSSimon Moll }
328c574c54eSSimon Moll 
legalizeInternalVectorOp(SDValue Op,SelectionDAG & DAG) const3297d926b71SSimon Moll SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
3307d926b71SSimon Moll                                                    SelectionDAG &DAG) const {
331c574c54eSSimon Moll   LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
3327d926b71SSimon Moll   VECustomDAG CDAG(DAG, Op);
3334fd77129SSimon Moll 
334c574c54eSSimon Moll   // Dispatch to specialized legalization functions.
335c574c54eSSimon Moll   switch (Op->getOpcode()) {
336c574c54eSSimon Moll   case VEISD::VVP_LOAD:
337c574c54eSSimon Moll   case VEISD::VVP_STORE:
338c574c54eSSimon Moll     return legalizeInternalLoadStoreOp(Op, CDAG);
339c574c54eSSimon Moll   }
340c574c54eSSimon Moll 
3414fd77129SSimon Moll   EVT IdiomVT = Op.getValueType();
3424fd77129SSimon Moll   if (isPackedVectorType(IdiomVT) &&
3434fd77129SSimon Moll       !supportsPackedMode(Op.getOpcode(), IdiomVT))
3444fd77129SSimon Moll     return splitVectorOp(Op, CDAG);
3454fd77129SSimon Moll 
3467d926b71SSimon Moll   // TODO: Implement odd/even splitting.
3477d926b71SSimon Moll   return legalizePackedAVL(Op, CDAG);
3487d926b71SSimon Moll }
3497d926b71SSimon Moll 
splitVectorOp(SDValue Op,VECustomDAG & CDAG) const3504fd77129SSimon Moll SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
3514fd77129SSimon Moll   MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
3524fd77129SSimon Moll 
3534fd77129SSimon Moll   auto AVLPos = getAVLPos(Op->getOpcode());
3544fd77129SSimon Moll   auto MaskPos = getMaskPos(Op->getOpcode());
3554fd77129SSimon Moll 
3564fd77129SSimon Moll   SDValue PackedMask = getNodeMask(Op);
3574fd77129SSimon Moll   auto AVLPair = getAnnotatedNodeAVL(Op);
3584fd77129SSimon Moll   SDValue PackedAVL = AVLPair.first;
3594fd77129SSimon Moll   assert(!AVLPair.second && "Expecting non pack-legalized oepration");
3604fd77129SSimon Moll 
3614fd77129SSimon Moll   // request the parts
3624fd77129SSimon Moll   SDValue PartOps[2];
3634fd77129SSimon Moll 
3644fd77129SSimon Moll   SDValue UpperPartAVL; // we will use this for packing things back together
3654fd77129SSimon Moll   for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
3664fd77129SSimon Moll     // VP ops already have an explicit mask and AVL. When expanding from non-VP
3674fd77129SSimon Moll     // attach those additional inputs here.
3684fd77129SSimon Moll     auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
3694fd77129SSimon Moll 
3704fd77129SSimon Moll     if (Part == PackElem::Hi)
3714fd77129SSimon Moll       UpperPartAVL = SplitTM.AVL;
3724fd77129SSimon Moll 
3734fd77129SSimon Moll     // Attach non-predicating value operands
3744fd77129SSimon Moll     SmallVector<SDValue, 4> OpVec;
3754fd77129SSimon Moll     for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
3764fd77129SSimon Moll       if (AVLPos && ((int)i) == *AVLPos)
3774fd77129SSimon Moll         continue;
3784fd77129SSimon Moll       if (MaskPos && ((int)i) == *MaskPos)
3794fd77129SSimon Moll         continue;
3804fd77129SSimon Moll 
3814fd77129SSimon Moll       // Value operand
3824fd77129SSimon Moll       auto PackedOperand = Op.getOperand(i);
3834fd77129SSimon Moll       auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
3844fd77129SSimon Moll       SDValue PartV =
3854fd77129SSimon Moll           CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
3864fd77129SSimon Moll       OpVec.push_back(PartV);
3874fd77129SSimon Moll     }
3884fd77129SSimon Moll 
3894fd77129SSimon Moll     // Add predicating args and generate part node.
3904fd77129SSimon Moll     OpVec.push_back(SplitTM.Mask);
3914fd77129SSimon Moll     OpVec.push_back(SplitTM.AVL);
3924fd77129SSimon Moll     // Emit legal VVP nodes.
3934fd77129SSimon Moll     PartOps[(int)Part] =
3944fd77129SSimon Moll         CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
3954fd77129SSimon Moll   }
3964fd77129SSimon Moll 
3974fd77129SSimon Moll   // Re-package vectors.
3984fd77129SSimon Moll   return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
3994fd77129SSimon Moll                       PartOps[(int)PackElem::Hi], UpperPartAVL);
4004fd77129SSimon Moll }
4014fd77129SSimon Moll 
legalizePackedAVL(SDValue Op,VECustomDAG & CDAG) const4027d926b71SSimon Moll SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
4037d926b71SSimon Moll                                             VECustomDAG &CDAG) const {
4047d926b71SSimon Moll   LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
4057d926b71SSimon Moll   // Only required for VEC and VVP ops.
4067d926b71SSimon Moll   if (!isVVPOrVEC(Op->getOpcode()))
4077d926b71SSimon Moll     return Op;
4087d926b71SSimon Moll 
4097d926b71SSimon Moll   // Operation already has a legal AVL.
4107d926b71SSimon Moll   auto AVL = getNodeAVL(Op);
4117d926b71SSimon Moll   if (isLegalAVL(AVL))
4127d926b71SSimon Moll     return Op;
4137d926b71SSimon Moll 
4147d926b71SSimon Moll   // Half and round up EVL for 32bit element types.
4157d926b71SSimon Moll   SDValue LegalAVL = AVL;
416c574c54eSSimon Moll   MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
417c574c54eSSimon Moll   if (isPackedVectorType(IdiomVT)) {
4187d926b71SSimon Moll     assert(maySafelyIgnoreMask(Op) &&
4197d926b71SSimon Moll            "TODO Shift predication from EVL into Mask");
4207d926b71SSimon Moll 
4217d926b71SSimon Moll     if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
4227d926b71SSimon Moll       LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
4237d926b71SSimon Moll     } else {
4247d926b71SSimon Moll       auto ConstOne = CDAG.getConstant(1, MVT::i32);
4257d926b71SSimon Moll       auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
4267d926b71SSimon Moll       LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
4277d926b71SSimon Moll     }
4287d926b71SSimon Moll   }
4297d926b71SSimon Moll 
4307d926b71SSimon Moll   SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
4317d926b71SSimon Moll 
4327d926b71SSimon Moll   // Copy the operand list.
4337d926b71SSimon Moll   int NumOp = Op->getNumOperands();
4347d926b71SSimon Moll   auto AVLPos = getAVLPos(Op->getOpcode());
4357d926b71SSimon Moll   std::vector<SDValue> FixedOperands;
4367d926b71SSimon Moll   for (int i = 0; i < NumOp; ++i) {
4377d926b71SSimon Moll     if (AVLPos && (i == *AVLPos)) {
4387d926b71SSimon Moll       FixedOperands.push_back(AnnotatedLegalAVL);
4397d926b71SSimon Moll       continue;
4407d926b71SSimon Moll     }
4417d926b71SSimon Moll     FixedOperands.push_back(Op->getOperand(i));
4427d926b71SSimon Moll   }
4437d926b71SSimon Moll 
4447d926b71SSimon Moll   // Clone the operation with fixed operands.
4457d926b71SSimon Moll   auto Flags = Op->getFlags();
4467d926b71SSimon Moll   SDValue NewN =
4477d926b71SSimon Moll       CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
4487d926b71SSimon Moll   return NewN;
4497d926b71SSimon Moll }
450