17d926b71SSimon Moll //===-- VVPISelLowering.cpp - VE DAG Lowering Implementation --------------===//
27d926b71SSimon Moll //
37d926b71SSimon Moll // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
47d926b71SSimon Moll // See https://llvm.org/LICENSE.txt for license information.
57d926b71SSimon Moll // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
67d926b71SSimon Moll //
77d926b71SSimon Moll //===----------------------------------------------------------------------===//
87d926b71SSimon Moll //
97d926b71SSimon Moll // This file implements the lowering and legalization of vector instructions to
107d926b71SSimon Moll // VVP_*layer SDNodes.
117d926b71SSimon Moll //
127d926b71SSimon Moll //===----------------------------------------------------------------------===//
137d926b71SSimon Moll
147d926b71SSimon Moll #include "VECustomDAG.h"
157d926b71SSimon Moll #include "VEISelLowering.h"
167d926b71SSimon Moll
177d926b71SSimon Moll using namespace llvm;
187d926b71SSimon Moll
197d926b71SSimon Moll #define DEBUG_TYPE "ve-lower"
207d926b71SSimon Moll
splitMaskArithmetic(SDValue Op,SelectionDAG & DAG) const21606320edSSimon Moll SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,
22606320edSSimon Moll SelectionDAG &DAG) const {
23606320edSSimon Moll VECustomDAG CDAG(DAG, Op);
24606320edSSimon Moll SDValue AVL =
25606320edSSimon Moll CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);
26606320edSSimon Moll SDValue A = Op->getOperand(0);
27606320edSSimon Moll SDValue B = Op->getOperand(1);
28606320edSSimon Moll SDValue LoA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Lo, AVL);
29606320edSSimon Moll SDValue HiA = CDAG.getUnpack(MVT::v256i1, A, PackElem::Hi, AVL);
30606320edSSimon Moll SDValue LoB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Lo, AVL);
31606320edSSimon Moll SDValue HiB = CDAG.getUnpack(MVT::v256i1, B, PackElem::Hi, AVL);
32606320edSSimon Moll unsigned Opc = Op.getOpcode();
33606320edSSimon Moll auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});
34606320edSSimon Moll auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});
35606320edSSimon Moll return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);
36606320edSSimon Moll }
37606320edSSimon Moll
lowerToVVP(SDValue Op,SelectionDAG & DAG) const38606320edSSimon Moll SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
39606320edSSimon Moll // Can we represent this as a VVP node.
40606320edSSimon Moll const unsigned Opcode = Op->getOpcode();
41606320edSSimon Moll auto VVPOpcodeOpt = getVVPOpcode(Opcode);
42a7938c74SKazu Hirata if (!VVPOpcodeOpt)
43606320edSSimon Moll return SDValue();
4421c4dc79SFangrui Song unsigned VVPOpcode = *VVPOpcodeOpt;
45606320edSSimon Moll const bool FromVP = ISD::isVPOpcode(Opcode);
46606320edSSimon Moll
47606320edSSimon Moll // The representative and legalized vector type of this operation.
48606320edSSimon Moll VECustomDAG CDAG(DAG, Op);
499ebaec46SSimon Moll // Dispatch to complex lowering functions.
509ebaec46SSimon Moll switch (VVPOpcode) {
519ebaec46SSimon Moll case VEISD::VVP_LOAD:
529ebaec46SSimon Moll case VEISD::VVP_STORE:
539ebaec46SSimon Moll return lowerVVP_LOAD_STORE(Op, CDAG);
54a5f12623SSimon Moll case VEISD::VVP_GATHER:
55a5f12623SSimon Moll case VEISD::VVP_SCATTER:
56a5f12623SSimon Moll return lowerVVP_GATHER_SCATTER(Op, CDAG);
57a5f12623SSimon Moll }
589ebaec46SSimon Moll
59f318d1e2SSimon Moll EVT OpVecVT = *getIdiomaticVectorType(Op.getNode());
60606320edSSimon Moll EVT LegalVecVT = getTypeToTransformTo(*DAG.getContext(), OpVecVT);
61606320edSSimon Moll auto Packing = getTypePacking(LegalVecVT.getSimpleVT());
62606320edSSimon Moll
63606320edSSimon Moll SDValue AVL;
64606320edSSimon Moll SDValue Mask;
65606320edSSimon Moll
66606320edSSimon Moll if (FromVP) {
67606320edSSimon Moll // All upstream VP SDNodes always have a mask and avl.
68606320edSSimon Moll auto MaskIdx = ISD::getVPMaskIdx(Opcode);
69606320edSSimon Moll auto AVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode);
70606320edSSimon Moll if (MaskIdx)
71606320edSSimon Moll Mask = Op->getOperand(*MaskIdx);
72606320edSSimon Moll if (AVLIdx)
73606320edSSimon Moll AVL = Op->getOperand(*AVLIdx);
74606320edSSimon Moll }
75606320edSSimon Moll
76606320edSSimon Moll // Materialize default mask and avl.
77606320edSSimon Moll if (!AVL)
78606320edSSimon Moll AVL = CDAG.getConstant(OpVecVT.getVectorNumElements(), MVT::i32);
79606320edSSimon Moll if (!Mask)
80606320edSSimon Moll Mask = CDAG.getConstantMask(Packing, true);
81606320edSSimon Moll
82606320edSSimon Moll assert(LegalVecVT.isSimple());
8391fad116SSimon Moll if (isVVPUnaryOp(VVPOpcode))
8491fad116SSimon Moll return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});
8591fad116SSimon Moll if (isVVPBinaryOp(VVPOpcode))
86606320edSSimon Moll return CDAG.getNode(VVPOpcode, LegalVecVT,
87606320edSSimon Moll {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
88f318d1e2SSimon Moll if (isVVPReductionOp(VVPOpcode)) {
89f318d1e2SSimon Moll auto SrcHasStart = hasReductionStartParam(Op->getOpcode());
90f318d1e2SSimon Moll SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();
91f318d1e2SSimon Moll SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
92f318d1e2SSimon Moll return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,
93f318d1e2SSimon Moll VectorV, Mask, AVL, Op->getFlags());
94f318d1e2SSimon Moll }
95f318d1e2SSimon Moll
963297571eSSimon Moll switch (VVPOpcode) {
973297571eSSimon Moll default:
983297571eSSimon Moll llvm_unreachable("lowerToVVP called for unexpected SDNode.");
993297571eSSimon Moll case VEISD::VVP_FFMA: {
1003297571eSSimon Moll // VE has a swizzled operand order in FMA (compared to LLVM IR and
1013297571eSSimon Moll // SDNodes).
1023297571eSSimon Moll auto X = Op->getOperand(2);
1033297571eSSimon Moll auto Y = Op->getOperand(0);
1043297571eSSimon Moll auto Z = Op->getOperand(1);
1053297571eSSimon Moll return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL});
1063297571eSSimon Moll }
1073297571eSSimon Moll case VEISD::VVP_SELECT: {
108606320edSSimon Moll auto Mask = Op->getOperand(0);
109606320edSSimon Moll auto OnTrue = Op->getOperand(1);
110606320edSSimon Moll auto OnFalse = Op->getOperand(2);
111606320edSSimon Moll return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
112606320edSSimon Moll }
1133297571eSSimon Moll case VEISD::VVP_SETCC: {
114f318d1e2SSimon Moll EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
115606320edSSimon Moll auto LHS = Op->getOperand(0);
116606320edSSimon Moll auto RHS = Op->getOperand(1);
117606320edSSimon Moll auto Pred = Op->getOperand(2);
118f318d1e2SSimon Moll return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL});
119606320edSSimon Moll }
1203297571eSSimon Moll }
121606320edSSimon Moll }
122606320edSSimon Moll
lowerVVP_LOAD_STORE(SDValue Op,VECustomDAG & CDAG) const1239ebaec46SSimon Moll SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,
1249ebaec46SSimon Moll VECustomDAG &CDAG) const {
1259ebaec46SSimon Moll auto VVPOpc = *getVVPOpcode(Op->getOpcode());
1269ebaec46SSimon Moll const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
1279ebaec46SSimon Moll
1289ebaec46SSimon Moll // Shares.
1299ebaec46SSimon Moll SDValue BasePtr = getMemoryPtr(Op);
1309ebaec46SSimon Moll SDValue Mask = getNodeMask(Op);
1319ebaec46SSimon Moll SDValue Chain = getNodeChain(Op);
1329ebaec46SSimon Moll SDValue AVL = getNodeAVL(Op);
1339ebaec46SSimon Moll // Store specific.
1349ebaec46SSimon Moll SDValue Data = getStoredValue(Op);
1359ebaec46SSimon Moll // Load specific.
1369ebaec46SSimon Moll SDValue PassThru = getNodePassthru(Op);
1379ebaec46SSimon Moll
1386ac3d8efSSimon Moll SDValue StrideV = getLoadStoreStride(Op, CDAG);
1396ac3d8efSSimon Moll
1409ebaec46SSimon Moll auto DataVT = *getIdiomaticVectorType(Op.getNode());
1419ebaec46SSimon Moll auto Packing = getTypePacking(DataVT);
1429ebaec46SSimon Moll
1439ebaec46SSimon Moll // TODO: Infer lower AVL from mask.
1449ebaec46SSimon Moll if (!AVL)
1459ebaec46SSimon Moll AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
1469ebaec46SSimon Moll
1479ebaec46SSimon Moll // Default to the all-true mask.
1489ebaec46SSimon Moll if (!Mask)
1499ebaec46SSimon Moll Mask = CDAG.getConstantMask(Packing, true);
1509ebaec46SSimon Moll
1519ebaec46SSimon Moll if (IsLoad) {
1529ebaec46SSimon Moll MVT LegalDataVT = getLegalVectorType(
1539ebaec46SSimon Moll Packing, DataVT.getVectorElementType().getSimpleVT());
1549ebaec46SSimon Moll
1559ebaec46SSimon Moll auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},
1569ebaec46SSimon Moll {Chain, BasePtr, StrideV, Mask, AVL});
1579ebaec46SSimon Moll
1589ebaec46SSimon Moll if (!PassThru || PassThru->isUndef())
1599ebaec46SSimon Moll return NewLoadV;
1609ebaec46SSimon Moll
1619ebaec46SSimon Moll // Convert passthru to an explicit select node.
1629ebaec46SSimon Moll SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,
1639ebaec46SSimon Moll {NewLoadV, PassThru, Mask, AVL});
1649ebaec46SSimon Moll SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
1659ebaec46SSimon Moll
1669ebaec46SSimon Moll // Merge them back into one node.
1679ebaec46SSimon Moll return CDAG.getMergeValues({DataV, NewLoadChainV});
1689ebaec46SSimon Moll }
1699ebaec46SSimon Moll
1709ebaec46SSimon Moll // VVP_STORE
1719ebaec46SSimon Moll assert(VVPOpc == VEISD::VVP_STORE);
172*2e239565SKazushi (Jam) Marukawa if (getTypeAction(*CDAG.getDAG()->getContext(), Data.getValueType()) !=
173*2e239565SKazushi (Jam) Marukawa TargetLowering::TypeLegal)
174*2e239565SKazushi (Jam) Marukawa // Doesn't lower store instruction if an operand is not lowered yet.
175*2e239565SKazushi (Jam) Marukawa // If it isn't, return SDValue(). In this way, LLVM will try to lower
176*2e239565SKazushi (Jam) Marukawa // store instruction again after lowering all operands.
177*2e239565SKazushi (Jam) Marukawa return SDValue();
1789ebaec46SSimon Moll return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
1799ebaec46SSimon Moll {Chain, Data, BasePtr, StrideV, Mask, AVL});
1809ebaec46SSimon Moll }
1819ebaec46SSimon Moll
splitPackedLoadStore(SDValue Op,VECustomDAG & CDAG) const182c574c54eSSimon Moll SDValue VETargetLowering::splitPackedLoadStore(SDValue Op,
183c574c54eSSimon Moll VECustomDAG &CDAG) const {
184c574c54eSSimon Moll auto VVPOC = *getVVPOpcode(Op.getOpcode());
185c574c54eSSimon Moll assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
186c574c54eSSimon Moll
187c574c54eSSimon Moll MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
188c574c54eSSimon Moll assert(getTypePacking(DataVT) == Packing::Dense &&
189c574c54eSSimon Moll "Can only split packed load/store");
190c574c54eSSimon Moll MVT SplitDataVT = splitVectorType(DataVT);
191c574c54eSSimon Moll
1926cf1bd3aSKazu Hirata assert(!getNodePassthru(Op) &&
1936cf1bd3aSKazu Hirata "Should have been folded in lowering to VVP layer");
194c574c54eSSimon Moll
195c574c54eSSimon Moll // Analyze the operation
196c574c54eSSimon Moll SDValue PackedMask = getNodeMask(Op);
197c574c54eSSimon Moll SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;
198c574c54eSSimon Moll SDValue PackPtr = getMemoryPtr(Op);
199c574c54eSSimon Moll SDValue PackData = getStoredValue(Op);
200c574c54eSSimon Moll SDValue PackStride = getLoadStoreStride(Op, CDAG);
201c574c54eSSimon Moll
202c574c54eSSimon Moll unsigned ChainResIdx = PackData ? 0 : 1;
203c574c54eSSimon Moll
204c574c54eSSimon Moll SDValue PartOps[2];
205c574c54eSSimon Moll
206c574c54eSSimon Moll SDValue UpperPartAVL; // we will use this for packing things back together
207c574c54eSSimon Moll for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
208c574c54eSSimon Moll // VP ops already have an explicit mask and AVL. When expanding from non-VP
209c574c54eSSimon Moll // attach those additional inputs here.
210c574c54eSSimon Moll auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
211c574c54eSSimon Moll
212c574c54eSSimon Moll // Keep track of the (higher) lvl.
213c574c54eSSimon Moll if (Part == PackElem::Hi)
214c574c54eSSimon Moll UpperPartAVL = SplitTM.AVL;
215c574c54eSSimon Moll
216c574c54eSSimon Moll // Attach non-predicating value operands
217c574c54eSSimon Moll SmallVector<SDValue, 4> OpVec;
218c574c54eSSimon Moll
219c574c54eSSimon Moll // Chain
220c574c54eSSimon Moll OpVec.push_back(getNodeChain(Op));
221c574c54eSSimon Moll
222c574c54eSSimon Moll // Data
223c574c54eSSimon Moll if (PackData) {
224c574c54eSSimon Moll SDValue PartData =
225c574c54eSSimon Moll CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
226c574c54eSSimon Moll OpVec.push_back(PartData);
227c574c54eSSimon Moll }
228c574c54eSSimon Moll
229c574c54eSSimon Moll // Ptr & Stride
230c574c54eSSimon Moll // Push (ptr + ElemBytes * <Part>, 2 * ElemBytes)
231c574c54eSSimon Moll // Stride info
232c574c54eSSimon Moll // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);
233c574c54eSSimon Moll OpVec.push_back(CDAG.getSplitPtrOffset(PackPtr, PackStride, Part));
234c574c54eSSimon Moll OpVec.push_back(CDAG.getSplitPtrStride(PackStride));
235c574c54eSSimon Moll
236c574c54eSSimon Moll // Add predicating args and generate part node
237c574c54eSSimon Moll OpVec.push_back(SplitTM.Mask);
238c574c54eSSimon Moll OpVec.push_back(SplitTM.AVL);
239c574c54eSSimon Moll
240c574c54eSSimon Moll if (PackData) {
241c574c54eSSimon Moll // Store
242c574c54eSSimon Moll PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);
243c574c54eSSimon Moll } else {
244c574c54eSSimon Moll // Load
245c574c54eSSimon Moll PartOps[(int)Part] =
246c574c54eSSimon Moll CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);
247c574c54eSSimon Moll }
248c574c54eSSimon Moll }
249c574c54eSSimon Moll
250c574c54eSSimon Moll // Merge the chains
251c574c54eSSimon Moll SDValue LowChain = SDValue(PartOps[(int)PackElem::Lo].getNode(), ChainResIdx);
252c574c54eSSimon Moll SDValue HiChain = SDValue(PartOps[(int)PackElem::Hi].getNode(), ChainResIdx);
253c574c54eSSimon Moll SDValue FusedChains =
254c574c54eSSimon Moll CDAG.getNode(ISD::TokenFactor, MVT::Other, {LowChain, HiChain});
255c574c54eSSimon Moll
256c574c54eSSimon Moll // Chain only [store]
257c574c54eSSimon Moll if (PackData)
258c574c54eSSimon Moll return FusedChains;
259c574c54eSSimon Moll
260c574c54eSSimon Moll // Re-pack into full packed vector result
261c574c54eSSimon Moll MVT PackedVT =
262c574c54eSSimon Moll getLegalVectorType(Packing::Dense, DataVT.getVectorElementType());
263c574c54eSSimon Moll SDValue PackedVals = CDAG.getPack(PackedVT, PartOps[(int)PackElem::Lo],
264c574c54eSSimon Moll PartOps[(int)PackElem::Hi], UpperPartAVL);
265c574c54eSSimon Moll
266c574c54eSSimon Moll return CDAG.getMergeValues({PackedVals, FusedChains});
267c574c54eSSimon Moll }
268c574c54eSSimon Moll
lowerVVP_GATHER_SCATTER(SDValue Op,VECustomDAG & CDAG) const269a5f12623SSimon Moll SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op,
270a5f12623SSimon Moll VECustomDAG &CDAG) const {
271a5f12623SSimon Moll EVT DataVT = *getIdiomaticVectorType(Op.getNode());
272a5f12623SSimon Moll auto Packing = getTypePacking(DataVT);
273a5f12623SSimon Moll MVT LegalDataVT =
274a5f12623SSimon Moll getLegalVectorType(Packing, DataVT.getVectorElementType().getSimpleVT());
275a5f12623SSimon Moll
276a5f12623SSimon Moll SDValue AVL = getAnnotatedNodeAVL(Op).first;
277a5f12623SSimon Moll SDValue Index = getGatherScatterIndex(Op);
278a5f12623SSimon Moll SDValue BasePtr = getMemoryPtr(Op);
279a5f12623SSimon Moll SDValue Mask = getNodeMask(Op);
280a5f12623SSimon Moll SDValue Chain = getNodeChain(Op);
281a5f12623SSimon Moll SDValue Scale = getGatherScatterScale(Op);
282a5f12623SSimon Moll SDValue PassThru = getNodePassthru(Op);
283a5f12623SSimon Moll SDValue StoredValue = getStoredValue(Op);
284a5f12623SSimon Moll if (PassThru && PassThru->isUndef())
285a5f12623SSimon Moll PassThru = SDValue();
286a5f12623SSimon Moll
287a5f12623SSimon Moll bool IsScatter = (bool)StoredValue;
288a5f12623SSimon Moll
289a5f12623SSimon Moll // TODO: Infer lower AVL from mask.
290a5f12623SSimon Moll if (!AVL)
291a5f12623SSimon Moll AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);
292a5f12623SSimon Moll
293a5f12623SSimon Moll // Default to the all-true mask.
294a5f12623SSimon Moll if (!Mask)
295a5f12623SSimon Moll Mask = CDAG.getConstantMask(Packing, true);
296a5f12623SSimon Moll
297a5f12623SSimon Moll SDValue AddressVec =
298a5f12623SSimon Moll CDAG.getGatherScatterAddress(BasePtr, Scale, Index, Mask, AVL);
299a5f12623SSimon Moll if (IsScatter)
300a5f12623SSimon Moll return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other,
301a5f12623SSimon Moll {Chain, StoredValue, AddressVec, Mask, AVL});
302a5f12623SSimon Moll
303a5f12623SSimon Moll // Gather.
304a5f12623SSimon Moll SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other},
305a5f12623SSimon Moll {Chain, AddressVec, Mask, AVL});
306a5f12623SSimon Moll
307a5f12623SSimon Moll if (!PassThru)
308a5f12623SSimon Moll return NewLoadV;
309a5f12623SSimon Moll
310a5f12623SSimon Moll // TODO: Use vvp_select
311a5f12623SSimon Moll SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT,
312a5f12623SSimon Moll {NewLoadV, PassThru, Mask, AVL});
313a5f12623SSimon Moll SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);
314a5f12623SSimon Moll return CDAG.getMergeValues({DataV, NewLoadChainV});
315a5f12623SSimon Moll }
316a5f12623SSimon Moll
legalizeInternalLoadStoreOp(SDValue Op,VECustomDAG & CDAG) const317c574c54eSSimon Moll SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,
318c574c54eSSimon Moll VECustomDAG &CDAG) const {
319c574c54eSSimon Moll LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);
320c574c54eSSimon Moll MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
321c574c54eSSimon Moll
322c574c54eSSimon Moll // TODO: Recognize packable load,store.
323c574c54eSSimon Moll if (isPackedVectorType(DataVT))
324c574c54eSSimon Moll return splitPackedLoadStore(Op, CDAG);
325c574c54eSSimon Moll
326c574c54eSSimon Moll return legalizePackedAVL(Op, CDAG);
327c574c54eSSimon Moll }
328c574c54eSSimon Moll
legalizeInternalVectorOp(SDValue Op,SelectionDAG & DAG) const3297d926b71SSimon Moll SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,
3307d926b71SSimon Moll SelectionDAG &DAG) const {
331c574c54eSSimon Moll LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);
3327d926b71SSimon Moll VECustomDAG CDAG(DAG, Op);
3334fd77129SSimon Moll
334c574c54eSSimon Moll // Dispatch to specialized legalization functions.
335c574c54eSSimon Moll switch (Op->getOpcode()) {
336c574c54eSSimon Moll case VEISD::VVP_LOAD:
337c574c54eSSimon Moll case VEISD::VVP_STORE:
338c574c54eSSimon Moll return legalizeInternalLoadStoreOp(Op, CDAG);
339c574c54eSSimon Moll }
340c574c54eSSimon Moll
3414fd77129SSimon Moll EVT IdiomVT = Op.getValueType();
3424fd77129SSimon Moll if (isPackedVectorType(IdiomVT) &&
3434fd77129SSimon Moll !supportsPackedMode(Op.getOpcode(), IdiomVT))
3444fd77129SSimon Moll return splitVectorOp(Op, CDAG);
3454fd77129SSimon Moll
3467d926b71SSimon Moll // TODO: Implement odd/even splitting.
3477d926b71SSimon Moll return legalizePackedAVL(Op, CDAG);
3487d926b71SSimon Moll }
3497d926b71SSimon Moll
splitVectorOp(SDValue Op,VECustomDAG & CDAG) const3504fd77129SSimon Moll SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {
3514fd77129SSimon Moll MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());
3524fd77129SSimon Moll
3534fd77129SSimon Moll auto AVLPos = getAVLPos(Op->getOpcode());
3544fd77129SSimon Moll auto MaskPos = getMaskPos(Op->getOpcode());
3554fd77129SSimon Moll
3564fd77129SSimon Moll SDValue PackedMask = getNodeMask(Op);
3574fd77129SSimon Moll auto AVLPair = getAnnotatedNodeAVL(Op);
3584fd77129SSimon Moll SDValue PackedAVL = AVLPair.first;
3594fd77129SSimon Moll assert(!AVLPair.second && "Expecting non pack-legalized oepration");
3604fd77129SSimon Moll
3614fd77129SSimon Moll // request the parts
3624fd77129SSimon Moll SDValue PartOps[2];
3634fd77129SSimon Moll
3644fd77129SSimon Moll SDValue UpperPartAVL; // we will use this for packing things back together
3654fd77129SSimon Moll for (PackElem Part : {PackElem::Hi, PackElem::Lo}) {
3664fd77129SSimon Moll // VP ops already have an explicit mask and AVL. When expanding from non-VP
3674fd77129SSimon Moll // attach those additional inputs here.
3684fd77129SSimon Moll auto SplitTM = CDAG.getTargetSplitMask(PackedMask, PackedAVL, Part);
3694fd77129SSimon Moll
3704fd77129SSimon Moll if (Part == PackElem::Hi)
3714fd77129SSimon Moll UpperPartAVL = SplitTM.AVL;
3724fd77129SSimon Moll
3734fd77129SSimon Moll // Attach non-predicating value operands
3744fd77129SSimon Moll SmallVector<SDValue, 4> OpVec;
3754fd77129SSimon Moll for (unsigned i = 0; i < Op.getNumOperands(); ++i) {
3764fd77129SSimon Moll if (AVLPos && ((int)i) == *AVLPos)
3774fd77129SSimon Moll continue;
3784fd77129SSimon Moll if (MaskPos && ((int)i) == *MaskPos)
3794fd77129SSimon Moll continue;
3804fd77129SSimon Moll
3814fd77129SSimon Moll // Value operand
3824fd77129SSimon Moll auto PackedOperand = Op.getOperand(i);
3834fd77129SSimon Moll auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());
3844fd77129SSimon Moll SDValue PartV =
3854fd77129SSimon Moll CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
3864fd77129SSimon Moll OpVec.push_back(PartV);
3874fd77129SSimon Moll }
3884fd77129SSimon Moll
3894fd77129SSimon Moll // Add predicating args and generate part node.
3904fd77129SSimon Moll OpVec.push_back(SplitTM.Mask);
3914fd77129SSimon Moll OpVec.push_back(SplitTM.AVL);
3924fd77129SSimon Moll // Emit legal VVP nodes.
3934fd77129SSimon Moll PartOps[(int)Part] =
3944fd77129SSimon Moll CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
3954fd77129SSimon Moll }
3964fd77129SSimon Moll
3974fd77129SSimon Moll // Re-package vectors.
3984fd77129SSimon Moll return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],
3994fd77129SSimon Moll PartOps[(int)PackElem::Hi], UpperPartAVL);
4004fd77129SSimon Moll }
4014fd77129SSimon Moll
legalizePackedAVL(SDValue Op,VECustomDAG & CDAG) const4027d926b71SSimon Moll SDValue VETargetLowering::legalizePackedAVL(SDValue Op,
4037d926b71SSimon Moll VECustomDAG &CDAG) const {
4047d926b71SSimon Moll LLVM_DEBUG(dbgs() << "::legalizePackedAVL\n";);
4057d926b71SSimon Moll // Only required for VEC and VVP ops.
4067d926b71SSimon Moll if (!isVVPOrVEC(Op->getOpcode()))
4077d926b71SSimon Moll return Op;
4087d926b71SSimon Moll
4097d926b71SSimon Moll // Operation already has a legal AVL.
4107d926b71SSimon Moll auto AVL = getNodeAVL(Op);
4117d926b71SSimon Moll if (isLegalAVL(AVL))
4127d926b71SSimon Moll return Op;
4137d926b71SSimon Moll
4147d926b71SSimon Moll // Half and round up EVL for 32bit element types.
4157d926b71SSimon Moll SDValue LegalAVL = AVL;
416c574c54eSSimon Moll MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();
417c574c54eSSimon Moll if (isPackedVectorType(IdiomVT)) {
4187d926b71SSimon Moll assert(maySafelyIgnoreMask(Op) &&
4197d926b71SSimon Moll "TODO Shift predication from EVL into Mask");
4207d926b71SSimon Moll
4217d926b71SSimon Moll if (auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
4227d926b71SSimon Moll LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);
4237d926b71SSimon Moll } else {
4247d926b71SSimon Moll auto ConstOne = CDAG.getConstant(1, MVT::i32);
4257d926b71SSimon Moll auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});
4267d926b71SSimon Moll LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});
4277d926b71SSimon Moll }
4287d926b71SSimon Moll }
4297d926b71SSimon Moll
4307d926b71SSimon Moll SDValue AnnotatedLegalAVL = CDAG.annotateLegalAVL(LegalAVL);
4317d926b71SSimon Moll
4327d926b71SSimon Moll // Copy the operand list.
4337d926b71SSimon Moll int NumOp = Op->getNumOperands();
4347d926b71SSimon Moll auto AVLPos = getAVLPos(Op->getOpcode());
4357d926b71SSimon Moll std::vector<SDValue> FixedOperands;
4367d926b71SSimon Moll for (int i = 0; i < NumOp; ++i) {
4377d926b71SSimon Moll if (AVLPos && (i == *AVLPos)) {
4387d926b71SSimon Moll FixedOperands.push_back(AnnotatedLegalAVL);
4397d926b71SSimon Moll continue;
4407d926b71SSimon Moll }
4417d926b71SSimon Moll FixedOperands.push_back(Op->getOperand(i));
4427d926b71SSimon Moll }
4437d926b71SSimon Moll
4447d926b71SSimon Moll // Clone the operation with fixed operands.
4457d926b71SSimon Moll auto Flags = Op->getFlags();
4467d926b71SSimon Moll SDValue NewN =
4477d926b71SSimon Moll CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);
4487d926b71SSimon Moll return NewN;
4497d926b71SSimon Moll }
450