21 SDValue VETargetLowering::splitMaskArithmetic(SDValue Op,  in splitMaskArithmetic()  argument
23   VECustomDAG CDAG(DAG, Op);  in splitMaskArithmetic()
25       CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);  in splitMaskArithmetic()
26   SDValue A = Op->getOperand(0);  in splitMaskArithmetic()
27   SDValue B = Op->getOperand(1);  in splitMaskArithmetic()
32   unsigned Opc = Op.getOpcode();  in splitMaskArithmetic()
38 SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {  in lowerToVVP()  argument
40   const unsigned Opcode = Op->getOpcode();  in lowerToVVP()
48   VECustomDAG CDAG(DAG, Op);  in lowerToVVP()
53     return lowerVVP_LOAD_STORE(Op, CDAG);  in lowerToVVP()
56     return lowerVVP_GATHER_SCATTER(Op, CDAG);  in lowerToVVP()
59   EVT OpVecVT = *getIdiomaticVectorType(Op.getNode());  in lowerToVVP()
71       Mask = Op->getOperand(*MaskIdx);  in lowerToVVP()
73       AVL = Op->getOperand(*AVLIdx);  in lowerToVVP()
84     return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});  in lowerToVVP()
87                         {Op->getOperand(0), Op->getOperand(1), Mask, AVL});  in lowerToVVP()
89     auto SrcHasStart = hasReductionStartParam(Op->getOpcode());  in lowerToVVP()
90     SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();  in lowerToVVP()
91     SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);  in lowerToVVP()
92     return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,  in lowerToVVP()
93                                        VectorV, Mask, AVL, Op->getFlags());  in lowerToVVP()
102     auto X = Op->getOperand(2);  in lowerToVVP()
103     auto Y = Op->getOperand(0);  in lowerToVVP()
104     auto Z = Op->getOperand(1);  in lowerToVVP()
108     auto Mask = Op->getOperand(0);  in lowerToVVP()
109     auto OnTrue = Op->getOperand(1);  in lowerToVVP()
110     auto OnFalse = Op->getOperand(2);  in lowerToVVP()
114     EVT LegalResVT = getTypeToTransformTo(*DAG.getContext(), Op.getValueType());  in lowerToVVP()
115     auto LHS = Op->getOperand(0);  in lowerToVVP()
116     auto RHS = Op->getOperand(1);  in lowerToVVP()
117     auto Pred = Op->getOperand(2);  in lowerToVVP()
123 SDValue VETargetLowering::lowerVVP_LOAD_STORE(SDValue Op,  in lowerVVP_LOAD_STORE()  argument
125   auto VVPOpc = *getVVPOpcode(Op->getOpcode());  in lowerVVP_LOAD_STORE()
129   SDValue BasePtr = getMemoryPtr(Op);  in lowerVVP_LOAD_STORE()
130   SDValue Mask = getNodeMask(Op);  in lowerVVP_LOAD_STORE()
131   SDValue Chain = getNodeChain(Op);  in lowerVVP_LOAD_STORE()
132   SDValue AVL = getNodeAVL(Op);  in lowerVVP_LOAD_STORE()
134   SDValue Data = getStoredValue(Op);  in lowerVVP_LOAD_STORE()
136   SDValue PassThru = getNodePassthru(Op);  in lowerVVP_LOAD_STORE()
138   SDValue StrideV = getLoadStoreStride(Op, CDAG);  in lowerVVP_LOAD_STORE()
140   auto DataVT = *getIdiomaticVectorType(Op.getNode());  in lowerVVP_LOAD_STORE()
178   return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),  in lowerVVP_LOAD_STORE()
182 SDValue VETargetLowering::splitPackedLoadStore(SDValue Op,  in splitPackedLoadStore()  argument
184   auto VVPOC = *getVVPOpcode(Op.getOpcode());  in splitPackedLoadStore()
187   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();  in splitPackedLoadStore()
192   assert(!getNodePassthru(Op) &&  in splitPackedLoadStore()
196   SDValue PackedMask = getNodeMask(Op);  in splitPackedLoadStore()
197   SDValue PackedAVL = getAnnotatedNodeAVL(Op).first;  in splitPackedLoadStore()
198   SDValue PackPtr = getMemoryPtr(Op);  in splitPackedLoadStore()
199   SDValue PackData = getStoredValue(Op);  in splitPackedLoadStore()
200   SDValue PackStride = getLoadStoreStride(Op, CDAG);  in splitPackedLoadStore()
220     OpVec.push_back(getNodeChain(Op));  in splitPackedLoadStore()
232     // EVT DataVT = LegalizeVectorType(getMemoryDataVT(Op), Op, DAG, Mode);  in splitPackedLoadStore()
269 SDValue VETargetLowering::lowerVVP_GATHER_SCATTER(SDValue Op,  in lowerVVP_GATHER_SCATTER()  argument
271   EVT DataVT = *getIdiomaticVectorType(Op.getNode());  in lowerVVP_GATHER_SCATTER()
276   SDValue AVL = getAnnotatedNodeAVL(Op).first;  in lowerVVP_GATHER_SCATTER()
277   SDValue Index = getGatherScatterIndex(Op);  in lowerVVP_GATHER_SCATTER()
278   SDValue BasePtr = getMemoryPtr(Op);  in lowerVVP_GATHER_SCATTER()
279   SDValue Mask = getNodeMask(Op);  in lowerVVP_GATHER_SCATTER()
280   SDValue Chain = getNodeChain(Op);  in lowerVVP_GATHER_SCATTER()
281   SDValue Scale = getGatherScatterScale(Op);  in lowerVVP_GATHER_SCATTER()
282   SDValue PassThru = getNodePassthru(Op);  in lowerVVP_GATHER_SCATTER()
283   SDValue StoredValue = getStoredValue(Op);  in lowerVVP_GATHER_SCATTER()
317 SDValue VETargetLowering::legalizeInternalLoadStoreOp(SDValue Op,  in legalizeInternalLoadStoreOp()  argument
320   MVT DataVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();  in legalizeInternalLoadStoreOp()
324     return splitPackedLoadStore(Op, CDAG);  in legalizeInternalLoadStoreOp()
326   return legalizePackedAVL(Op, CDAG);  in legalizeInternalLoadStoreOp()
329 SDValue VETargetLowering::legalizeInternalVectorOp(SDValue Op,  in legalizeInternalVectorOp()  argument
332   VECustomDAG CDAG(DAG, Op);  in legalizeInternalVectorOp()
335   switch (Op->getOpcode()) {  in legalizeInternalVectorOp()
338     return legalizeInternalLoadStoreOp(Op, CDAG);  in legalizeInternalVectorOp()
341   EVT IdiomVT = Op.getValueType();  in legalizeInternalVectorOp()
343       !supportsPackedMode(Op.getOpcode(), IdiomVT))  in legalizeInternalVectorOp()
344     return splitVectorOp(Op, CDAG);  in legalizeInternalVectorOp()
347   return legalizePackedAVL(Op, CDAG);  in legalizeInternalVectorOp()
350 SDValue VETargetLowering::splitVectorOp(SDValue Op, VECustomDAG &CDAG) const {  in splitVectorOp()  argument
351   MVT ResVT = splitVectorType(Op.getValue(0).getSimpleValueType());  in splitVectorOp()
353   auto AVLPos = getAVLPos(Op->getOpcode());  in splitVectorOp()
354   auto MaskPos = getMaskPos(Op->getOpcode());  in splitVectorOp()
356   SDValue PackedMask = getNodeMask(Op);  in splitVectorOp()
357   auto AVLPair = getAnnotatedNodeAVL(Op);  in splitVectorOp()
375     for (unsigned i = 0; i < Op.getNumOperands(); ++i) {  in splitVectorOp()
382       auto PackedOperand = Op.getOperand(i);  in splitVectorOp()
394         CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());  in splitVectorOp()
398   return CDAG.getPack(Op.getValueType(), PartOps[(int)PackElem::Lo],  in splitVectorOp()
402 SDValue VETargetLowering::legalizePackedAVL(SDValue Op,  in legalizePackedAVL()  argument
406   if (!isVVPOrVEC(Op->getOpcode()))  in legalizePackedAVL()
407     return Op;  in legalizePackedAVL()
410   auto AVL = getNodeAVL(Op);  in legalizePackedAVL()
412     return Op;  in legalizePackedAVL()
416   MVT IdiomVT = getIdiomaticVectorType(Op.getNode())->getSimpleVT();  in legalizePackedAVL()
418     assert(maySafelyIgnoreMask(Op) &&  in legalizePackedAVL()
433   int NumOp = Op->getNumOperands();  in legalizePackedAVL()
434   auto AVLPos = getAVLPos(Op->getOpcode());  in legalizePackedAVL()
441     FixedOperands.push_back(Op->getOperand(i));  in legalizePackedAVL()
445   auto Flags = Op->getFlags();  in legalizePackedAVL()
447       CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);  in legalizePackedAVL()