Lines Matching defs:VT
173 auto addRegClassForRVV = [this](MVT VT) {
177 if (VT.getVectorMinNumElements() < MinElts)
180 unsigned Size = VT.getSizeInBits().getKnownMinValue();
193 addRegisterClass(VT, RC);
196 for (MVT VT : BoolVecVTs)
197 addRegClassForRVV(VT);
198 for (MVT VT : IntVecVTs) {
199 if (VT.getVectorElementType() == MVT::i64 &&
202 addRegClassForRVV(VT);
206 for (MVT VT : F16VecVTs)
207 addRegClassForRVV(VT);
210 for (MVT VT : BF16VecVTs)
211 addRegClassForRVV(VT);
214 for (MVT VT : F32VecVTs)
215 addRegClassForRVV(VT);
218 for (MVT VT : F64VecVTs)
219 addRegClassForRVV(VT);
222 auto addRegClassForFixedVectors = [this](MVT VT) {
223 MVT ContainerVT = getContainerForFixedLengthVector(VT);
226 addRegisterClass(VT, TRI.getRegClass(RCID));
228 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
229 if (useRVVForFixedLengthVectorVT(VT))
230 addRegClassForFixedVectors(VT);
232 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
233 if (useRVVForFixedLengthVectorVT(VT))
234 addRegClassForFixedVectors(VT);
748 for (MVT VT : BoolVecVTs) {
749 if (!isTypeLegal(VT))
752 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
758 VT, Custom);
760 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
763 setOperationAction(ISD::SELECT, VT, Custom);
764 setOperationAction({ISD::SELECT_CC, ISD::VSELECT, ISD::VP_SELECT}, VT,
766 setOperationAction(ISD::VP_MERGE, VT, Custom);
768 setOperationAction({ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF}, VT,
771 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom);
774 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
778 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
789 VT, Custom);
790 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
796 setTruncStoreAction(VT, OtherVT, Expand);
797 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
803 VT, Custom);
805 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
806 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
808 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
810 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
811 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
814 ISD::VECTOR_SPLICE, VT,
815 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
818 for (MVT VT : IntVecVTs) {
819 if (!isTypeLegal(VT))
822 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
823 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
826 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
829 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
830 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
832 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
835 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
839 VT, Custom);
849 VT, Custom);
850 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
855 VT, Legal);
860 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
864 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
869 setOperationAction(IntegerVecReduceOps, VT, Custom);
871 setOperationAction(IntegerVPOps, VT, Custom);
873 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
876 VT, Custom);
881 VT, Custom);
885 VT, Custom);
887 setOperationAction(ISD::SELECT, VT, Custom);
888 setOperationAction(ISD::SELECT_CC, VT, Expand);
890 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
893 setTruncStoreAction(VT, OtherVT, Expand);
894 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
898 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
899 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
902 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
905 setOperationAction(ISD::BSWAP, VT, Legal);
906 setOperationAction(ISD::VP_BSWAP, VT, Custom);
908 setOperationAction({ISD::BSWAP, ISD::VP_BSWAP}, VT, Expand);
909 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
913 setOperationAction(ISD::BITREVERSE, VT, Legal);
914 setOperationAction(ISD::VP_BITREVERSE, VT, Custom);
917 VT, Custom);
919 setOperationAction({ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Expand);
920 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
923 VT, Expand);
925 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
927 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
932 VT, Custom);
936 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
939 for (MVT VT : VecTupleVTs) {
940 if (!isTypeLegal(VT))
943 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
995 const auto SetCommonVFPActions = [&](MVT VT) {
996 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1001 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1002 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1004 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1007 setCondCodeAction(VFPCCToExpand, VT, Expand);
1009 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal);
1010 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1015 VT, Custom);
1017 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1020 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1022 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1024 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1027 VT, Custom);
1032 VT, Custom);
1034 setOperationAction(ISD::SELECT, VT, Custom);
1035 setOperationAction(ISD::SELECT_CC, VT, Expand);
1039 VT, Custom);
1041 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1042 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1044 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom);
1046 setOperationAction(FloatingPointVPOps, VT, Custom);
1048 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1052 VT, Legal);
1057 VT, Custom);
1059 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1065 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1067 setTruncStoreAction(VT, SmallVT, Expand);
1068 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
1074 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1075 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1076 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1078 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1079 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1081 setOperationAction(ISD::SELECT_CC, VT, Expand);
1082 setOperationAction({ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, VT, Custom);
1088 VT, Custom);
1089 MVT EltVT = VT.getVectorElementType();
1093 VT, Custom);
1102 VT, Custom);
1104 setOperationAction(ISD::FNEG, VT, Expand);
1105 setOperationAction(ISD::FABS, VT, Expand);
1106 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1109 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1112 if (getLMUL(VT) == RISCVII::VLMUL::LMUL_8) {
1113 setOperationAction(ZvfhminZvfbfminPromoteOps, VT, Custom);
1114 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1116 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1117 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1118 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1123 for (MVT VT : F16VecVTs) {
1124 if (!isTypeLegal(VT))
1126 SetCommonVFPActions(VT);
1129 for (MVT VT : F16VecVTs) {
1130 if (!isTypeLegal(VT))
1132 SetCommonPromoteToF32Actions(VT);
1137 for (MVT VT : BF16VecVTs) {
1138 if (!isTypeLegal(VT))
1140 SetCommonPromoteToF32Actions(VT);
1145 for (MVT VT : F32VecVTs) {
1146 if (!isTypeLegal(VT))
1148 SetCommonVFPActions(VT);
1149 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1150 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1155 for (MVT VT : F64VecVTs) {
1156 if (!isTypeLegal(VT))
1158 SetCommonVFPActions(VT);
1159 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1160 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1161 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1166 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1167 if (!useRVVForFixedLengthVectorVT(VT))
1172 setOperationAction(Op, VT, Expand);
1174 setTruncStoreAction(VT, OtherVT, Expand);
1175 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1181 setOperationAction(ISD::UNDEF, VT, Custom);
1184 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1188 {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1192 VT, Custom);
1194 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1196 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1198 setOperationAction(ISD::SETCC, VT, Custom);
1200 setOperationAction(ISD::SELECT, VT, Custom);
1203 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1206 setOperationAction(ISD::BITCAST, VT, Custom);
1209 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1213 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1227 VT, Custom);
1228 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1231 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1234 if (VT.getVectorElementType() == MVT::i1) {
1237 VT, Custom);
1241 VT, Custom);
1243 setOperationAction(ISD::VP_MERGE, VT, Custom);
1245 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1246 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1255 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1256 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1257 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1261 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
1267 VT, Custom);
1272 VT, Custom);
1275 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
1277 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1280 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1281 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
1286 VT, Custom);
1288 setOperationAction(ISD::VSELECT, VT, Custom);
1291 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
1298 VT, Custom);
1300 setOperationAction(IntegerVPOps, VT, Custom);
1303 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
1308 VT, Custom);
1310 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
1312 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1315 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
1319 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1322 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1325 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1326 setTruncStoreAction(VT, InnerVT, Expand);
1329 if (!useRVVForFixedLengthVectorVT(VT))
1334 setOperationAction(Op, VT, Expand);
1338 setOperationAction(ISD::UNDEF, VT, Custom);
1344 VT, Custom);
1348 VT, Custom);
1352 VT, Custom);
1354 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1355 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1358 if (VT.getVectorElementType() == MVT::f16 &&
1360 setOperationAction(ISD::BITCAST, VT, Custom);
1361 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1363 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1365 setOperationAction({ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, VT,
1368 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1374 setOperationAction(ISD::FNEG, VT, Expand);
1375 setOperationAction(ISD::FABS, VT, Expand);
1376 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1377 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1383 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1384 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1388 if (VT.getVectorElementType() == MVT::bf16) {
1389 setOperationAction(ISD::BITCAST, VT, Custom);
1390 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1392 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1399 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1401 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1407 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1412 setOperationAction({ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1419 VT, Custom);
1423 VT, Custom);
1425 setCondCodeAction(VFPCCToExpand, VT, Expand);
1427 setOperationAction(ISD::SETCC, VT, Custom);
1428 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
1430 setOperationAction(ISD::BITCAST, VT, Custom);
1432 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1434 setOperationAction(FloatingPointVPOps, VT, Custom);
1442 VT, Custom);
1576 EVT VT) const {
1577 if (!VT.isVector())
1580 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1581 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1582 return VT.changeVectorElementTypeToInteger();
1616 bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
1618 VT.getVectorElementType() != MVT::i1 || !isTypeLegal(VT);
2028 EVT VT = Y.getValueType();
2031 if (VT.isVector())
2051 EVT VT) const {
2053 if (!VT.isVector() || !Subtarget.hasVInstructions())
2056 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2149 int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2154 if (VT == MVT::f16) {
2156 } else if (VT == MVT::f32) {
2158 } else if (VT == MVT::f64) {
2169 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2172 if (VT == MVT::f16)
2174 else if (VT == MVT::f32)
2176 else if (VT == MVT::f64)
2178 else if (VT == MVT::bf16)
2184 if (getLegalZfaFPImm(Imm, VT) >= 0)
2188 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2249 EVT VT) const {
2252 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2256 MVT PartVT = TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2262 RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2265 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2269 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2274 EVT VT) const {
2277 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2281 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2285 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
2288 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2359 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
2360 if (VT.isRISCVVectorTuple()) {
2361 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
2362 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
2364 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
2365 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
2367 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
2368 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
2370 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
2371 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
2373 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
2374 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
2376 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
2381 assert(VT.isScalableVector() && "Expecting a scalable vector type");
2382 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
2383 if (VT.getVectorElementType() == MVT::i1)
2424 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
2425 RISCVII::VLMUL LMUL = getLMUL(VT);
2447 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
2448 if (VT.isRISCVVectorTuple()) {
2449 unsigned NF = VT.getRISCVVectorTupleNumFields();
2451 std::max(1U, (unsigned)VT.getSizeInBits().getKnownMinValue() /
2487 if (VT.getVectorElementType() == MVT::i1)
2489 return getRegClassIDForLMUL(getLMUL(VT));
2545 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
2547 (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
2593 static bool useRVVForFixedLengthVectorVT(MVT VT,
2595 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
2603 if (VT.getFixedSizeInBits() > 1024 * 8)
2608 MVT EltVT = VT.getVectorElementType();
2617 if (VT.getVectorNumElements() > MinVLen)
2651 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
2658 if (!VT.isPow2VectorType())
2664 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
2665 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2668 // Return the largest legal scalable vector type that matches VT's element type.
2669 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
2672 assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
2673 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
2679 MVT EltVT = VT.getVectorElementType();
2696 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
2704 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
2706 return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
2710 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
2711 return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
2715 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
2717 assert(VT.isScalableVector() &&
2723 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
2726 // Shrink V so it's just big enough to maintain a VT's worth of data.
2727 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
2729 assert(VT.isFixedLengthVector() &&
2735 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
2822 EVT VT, unsigned DefinedValues) const {
2826 InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
2829 if (!VT.isVector())
2833 if (VT.isScalableVector()) {
2837 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT));
2843 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
2849 /// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
2852 InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
2853 return getLMULCost(VT) * getLMULCost(VT);
2856 /// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
2859 InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const {
2860 return getLMULCost(VT);
2864 /// for the type VT. (This does not cover the vslide1up or vslide1down
2867 InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const {
2868 return getLMULCost(VT);
2872 /// for the type VT. (This does not cover the vslide1up or vslide1down
2875 InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const {
2876 return getLMULCost(VT);
2982 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3081 MVT VT = Op.getSimpleValueType();
3082 assert(VT.isVector() && "Unexpected type");
3088 MVT ContainerVT = VT;
3089 if (VT.isFixedLengthVector()) {
3090 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3097 if (VT.isFixedLengthVector())
3102 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3176 if (!VT.isFixedLengthVector())
3179 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3189 MVT VT = Op.getSimpleValueType();
3193 MVT ContainerVT = VT;
3194 if (VT.isFixedLengthVector()) {
3195 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3199 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3281 if (VT.isFixedLengthVector())
3282 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3289 MVT VT = Op.getSimpleValueType();
3290 if (VT.isVector())
3302 const fltSemantics &FltSem = VT.getFltSemantics();
3307 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
3310 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
3317 MVT VT = Op.getSimpleValueType();
3318 assert(VT.isVector() && "Unexpected type");
3322 MVT ContainerVT = VT;
3324 if (VT.isFixedLengthVector()) {
3325 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3329 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3335 if (!VT.isFixedLengthVector())
3338 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3343 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
3350 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3355 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
3362 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3365 static MVT getLMUL1VT(MVT VT) {
3366 assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
3369 VT.getVectorElementType(),
3370 RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
3518 static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
3527 MVT EltTy = VT.getVectorElementType();
3536 // Check that we know Idx lies within VT
3537 if (!TypeSize::isKnownLE(SrcVT.getSizeInBits(), VT.getSizeInBits())) {
3539 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
3544 MVT ContainerVT = VT;
3545 if (VT.isFixedLengthVector())
3546 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3554 // Put Vec in a VT sized vector
3564 // We checked that Idx fits inside VT earlier
3565 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3568 if (VT.isFixedLengthVector())
3569 Gather = convertFromScalableVector(VT, Gather, DAG, Subtarget);
3583 MVT VT = Op.getSimpleValueType();
3584 assert(VT.isFixedLengthVector() && "Unexpected vector!");
3586 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3589 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3636 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
3650 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
3651 if (!VT.isFloatingPoint())
3655 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
3659 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
3665 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
3675 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
3677 DAG.getSplatBuildVector(VT, DL, V), Vec);
3689 MVT VT = Op.getSimpleValueType();
3690 assert(VT.isFixedLengthVector() && "Unexpected vector!");
3692 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3695 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3700 if (VT.getVectorElementType() == MVT::i1) {
3703 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
3708 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
3765 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
3770 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
3771 Vec = DAG.getBitcast(VT, Vec);
3778 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
3780 if (!VT.isFloatingPoint())
3784 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
3817 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
3841 if (VT.isFloatingPoint()) {
3843 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3851 const unsigned EltBitSize = VT.getScalarSizeInBits();
3852 if (VT.getSizeInBits() <= 32 &&
3854 MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
3861 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
3888 return DAG.getBitcast(VT, Vec);
3901 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
3958 return DAG.getBitcast(VT, Splat);
3968 if (EltBitSize > 8 && VT.isInteger() &&
3969 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
3971 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
3976 return convertFromScalableVector(VT, Res, DAG, Subtarget);
4011 MVT VT = Op.getSimpleValueType();
4012 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4013 MVT ElemVT = VT.getVectorElementType();
4022 unsigned NumElts = VT.getVectorNumElements();
4056 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4061 return DAG.getNode(ISD::BITCAST, DL, VT,
4067 MVT VT = Op.getSimpleValueType();
4068 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4070 MVT EltVT = VT.getVectorElementType();
4079 MVT IVT = VT.changeVectorElementType(MVT::i16);
4102 return DAG.getBitcast(VT, Res);
4109 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4111 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4113 if (VT.getVectorElementType() == MVT::i1) {
4117 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
4134 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4138 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
4140 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4142 if (!VT.isFloatingPoint())
4146 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4155 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4156 MVT ElemVT = VT.getVectorElementType();
4158 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4170 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4179 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4197 unsigned NumElts = VT.getVectorNumElements();
4223 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
4224 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
4227 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
4237 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
4275 assert((!VT.isFloatingPoint() ||
4276 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
4293 Vec = DAG.getSplatVector(VT, DL, V);
4306 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4307 if (!VT.isFloatingPoint())
4317 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4320 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
4324 Passthru = DAG.getUNDEF(VT);
4331 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4349 MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4352 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
4361 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4366 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4369 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
4376 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
4379 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
4382 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
4389 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
4393 Passthru = DAG.getUNDEF(VT);
4395 MVT EltVT = VT.getVectorElementType();
4398 if (VT.isFloatingPoint()) {
4406 MVT IVT = VT.changeVectorElementType(MVT::i16);
4410 return DAG.getNode(ISD::BITCAST, DL, VT, Splat);
4412 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
4424 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
4431 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
4435 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
4442 static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
4445 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
4448 SDValue Passthru = DAG.getUNDEF(VT);
4455 VT.getVectorElementType()) {
4464 if (ExtractedContainerVT.bitsLE(VT))
4465 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru,
4467 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal,
4473 if (VT.isFloatingPoint())
4474 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT,
4475 DAG.getUNDEF(VT), Scalar, VL);
4480 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
4482 VT, DL, DAG, Subtarget);
4491 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
4496 static SDValue getSingleShuffleSrc(MVT VT, MVT ContainerVT, SDValue V1,
4514 unsigned NumElts = VT.getVectorNumElements();
4532 static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
4535 if (VT.getScalarSizeInBits() >= Subtarget.getELen())
4539 int NumElts = VT.getVectorNumElements();
4638 // 2, 4, 8 and the integer type Factor-times larger than VT's
4642 static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
4645 unsigned EltBits = VT.getScalarSizeInBits();
4657 MVT IntVT = VT.changeVectorElementTypeToInteger();
4660 return DAG.getBitcast(VT, Res);
4675 static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
4735 ISD::EXTRACT_SUBVECTOR, DL, VT,
4752 static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
4757 unsigned NumElts = VT.getVectorNumElements();
4768 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4769 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
4791 return convertFromScalableVector(VT, Res, DAG, Subtarget);
4796 static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
4823 const unsigned NumElts = VT.getVectorNumElements();
4833 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4834 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4838 if (VT.getVectorElementType() == MVT::bf16 ||
4839 (VT.getVectorElementType() == MVT::f16 &&
4850 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4854 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
4855 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
4856 if (!VT.isFloatingPoint())
4862 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4902 MVT VT = V.getSimpleValueType();
4903 unsigned EltBits = VT.getScalarSizeInBits();
4904 ElementCount EC = VT.getVectorElementCount();
4905 V = DAG.getBitcast(VT.changeTypeToInteger(), V);
4916 MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(),
5021 MVT VT = SVN->getSimpleValueType(0);
5023 unsigned NumElts = VT.getVectorNumElements();
5025 assert(VT.getVectorElementType() == MVT::i1);
5064 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
5075 EVT VT = SVN->getValueType(0);
5076 unsigned NumElts = VT.getVectorNumElements();
5077 unsigned EltSizeInBits = VT.getScalarSizeInBits();
5097 EVT VT = SVN->getValueType(0);
5114 return DAG.getBitcast(VT, Rotate);
5124 MVT VT = SVN->getSimpleValueType(0);
5132 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
5142 MVT ElemVT = VT.getVectorElementType();
5145 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5242 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5282 MVT VT = SVN->getSimpleValueType(0);
5311 MVT MaskVT = VT.changeVectorElementType(MVT::i1);
5313 SDValue Select = DAG.getNode(ISD::VSELECT, DL, VT, SelectMask,
5325 return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
5336 MVT VT = Op.getSimpleValueType();
5337 MVT ScalarVT = VT.getVectorElementType();
5351 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(ElementSize * 2)
5353 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
5358 return DAG.getBitcast(VT, DAG.getVectorShuffle(NewVT, DL, V0, V1, NewMask));
5367 MVT VT = Op.getSimpleValueType();
5368 unsigned NumElts = VT.getVectorNumElements();
5371 if (VT.getVectorElementType() == MVT::i1) {
5380 MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
5385 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
5389 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5391 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5396 MVT SVT = VT.getVectorElementType();
5434 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5466 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
5474 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
5486 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
5490 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
5530 return convertFromScalableVector(VT, Res, DAG, Subtarget);
5534 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
5542 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
5543 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
5549 if (SDValue Src = getSingleShuffleSrc(VT, ContainerVT, V1, V2))
5550 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
5556 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
5562 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget)) {
5564 MVT HalfVT = VT.getHalfNumVectorElementsVT();
5626 return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
5627 DAG.getUNDEF(VT));
5632 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
5633 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
5639 MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
5647 if (VT.getScalarSizeInBits() == 8 &&
5659 MVT IndexVT = VT.changeTypeToInteger();
5693 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
5713 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS.data(), VT) &&
5714 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS.data(), VT) &&
5733 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
5734 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
5747 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
5748 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
5751 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
5753 if (!isTypeLegal(VT))
5757 if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
5760 MVT SVT = VT.getSimpleVT();
5776 MVT VT = Op.getSimpleValueType();
5777 unsigned EltSize = VT.getScalarSizeInBits();
5780 MVT ContainerVT = VT;
5785 if (VT.isFixedLengthVector())
5795 if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
5797 MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
5808 SDValue Neg = DAG.getNegative(Src, DL, VT);
5809 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
5811 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT),
5813 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL);
5818 if (FloatVT.bitsGT(VT)) {
5825 if (VT.isFixedLengthVector()) {
5826 ContainerVT = getContainerForFixedLengthVector(VT);
5830 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5837 if (VT.isFixedLengthVector())
5850 Exp = DAG.getVPZExtOrTrunc(DL, VT, Exp, Mask, VL);
5854 if (IntVT.bitsLT(VT))
5855 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
5856 else if (IntVT.bitsGT(VT))
5857 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
5864 return DAG.getNode(ISD::SUB, DL, VT, Exp,
5865 DAG.getConstant(ExponentBias, DL, VT));
5867 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp,
5868 DAG.getConstant(ExponentBias, DL, VT), Mask, VL);
5875 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp,
5878 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
5883 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
5885 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res,
5886 DAG.getConstant(EltSize, DL, VT), Mask, VL);
5943 MVT VT = Op.getSimpleValueType();
5944 unsigned EltSizeBits = VT.getScalarSizeInBits();
5948 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
5954 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
5974 MVT VT = StoredVal.getSimpleValueType();
5975 unsigned EltSizeBits = VT.getScalarSizeInBits();
5979 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
5990 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
6030 MVT VT = Op.getSimpleValueType();
6035 int Index = getLegalZfaFPImm(Imm, VT);
6039 Index = getLegalZfaFPImm(-Imm, VT);
6050 DAG.getNode(RISCVISD::FLI, DL, VT,
6055 return DAG.getNode(ISD::FNEG, DL, VT, Const);
6090 MVT VT = Op.getSimpleValueType();
6119 if (VT.isVector()) {
6123 if (VT.isScalableVector()) {
6133 return DAG.getSetCC(DL, VT, FPCLASS,
6138 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
6143 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6165 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
6177 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
6184 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
6192 MVT VT = Op.getSimpleValueType();
6197 if (!VT.isVector()) {
6208 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
6214 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
6219 return DAG.getNode(Opc, DL, VT, NewX, NewY);
6226 MVT ContainerVT = VT;
6227 if (VT.isFixedLengthVector()) {
6228 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
6236 if (VT.isFixedLengthVector())
6241 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6268 if (VT.isFixedLengthVector())
6269 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
6280 MVT VT = Op.getSimpleValueType();
6281 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
6293 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, Logic);
6301 MVT VT = Op.getSimpleValueType();
6302 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
6353 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
6714 EVT VT = Op.getValueType();
6719 ((VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
6720 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
6722 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, NewOp0);
6724 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
6729 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() &&
6738 if (!VT.isVector() && !Op0VT.isVector()) {
6739 if (isTypeLegal(VT) && isTypeLegal(Op0VT))
6744 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
6747 if (VT.isFixedLengthVector()) {
6759 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
6768 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
6769 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6773 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6787 MVT VT = Op.getSimpleValueType();
6788 if (VT.isFixedLengthVector()) {
6796 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
6797 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
6824 MVT VT = Op.getSimpleValueType();
6827 if (VT.getVectorElementType() == MVT::i1) {
6828 MVT WideVT = VT.changeVectorElementType(MVT::i8);
6830 return DAG.getNode(ISD::TRUNCATE, DL, VT, V);
6832 MVT ContainerVT = VT;
6833 if (VT.isFixedLengthVector())
6834 ContainerVT = getContainerForFixedLengthVector(VT);
6835 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6838 if (VT.isFloatingPoint()) {
6846 if (VT.isFixedLengthVector())
6847 V = convertFromScalableVector(VT, V, DAG, Subtarget);
6852 MVT VT = Op.getSimpleValueType();
6868 DAG.getConstant(3 - Log2, DL, VT));
6883 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
6955 MVT VT = Op.getSimpleValueType();
6956 if (VT.isScalarInteger())
6963 if (!VT.isVector())
6966 MVT EltVT = VT.getVectorElementType();
6979 VT.getVectorElementCount());
6988 return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
6994 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
7001 return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
7009 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
7015 return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first;
7018 return DAG.getFPExtendOrRound(Int2FP, DL, VT);
7024 VT.getVectorElementCount());
7029 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
7033 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
7038 if (!VT.isFixedLengthVector())
7072 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7077 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7083 SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget);
7087 return convertFromScalableVector(VT, Src, DAG, Subtarget);
7108 MVT VT = Op.getSimpleValueType();
7116 // fp_extend if the target VT is bigger than f32.
7117 if (VT != MVT::f32)
7118 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
7261 MVT VT = Op.getSimpleValueType();
7262 MVT EltVT = VT.getVectorElementType();
7273 MVT IVT = VT.changeVectorElementType(MVT::i16);
7274 return DAG.getNode(ISD::BITCAST, DL, VT,
7288 MVT VT = Op.getSimpleValueType();
7289 MVT ContainerVT = VT;
7290 if (VT.isFixedLengthVector())
7291 ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
7303 MVT HalfVT = VT.getHalfNumVectorElementsVT();
7310 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
7315 SDValue Vec = DAG.getUNDEF(VT);
7322 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
7429 EVT VT = Op.getValueType();
7437 return DAG.getSelect(DL, VT, Cond, True, False);
7442 MVT VT = Op.getSimpleValueType();
7461 return DAG.getConstant(0, DL, VT);
7465 DL, VT, LHS, DAG.getSignedConstant(Imm + 1, DL, OpVT), CCVal);
7466 return DAG.getLogicalNOT(DL, SetCC, VT);
7473 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
7537 EVT VT = Op->getValueType(0);
7546 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
7547 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
7548 return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
8202 MVT VT = N->getSimpleValueType(0);
8208 SDValue Neg = DAG.getNegative(CondV, DL, VT);
8209 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
8213 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
8214 DAG.getAllOnesConstant(DL, VT));
8215 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
8220 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
8221 DAG.getAllOnesConstant(DL, VT));
8222 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
8226 SDValue Neg = DAG.getNegative(CondV, DL, VT);
8227 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
8236 SDValue Neg = DAG.getNegative(CondV, DL, VT);
8237 return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
8252 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
8258 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
8305 EVT VT = BO->getValueType(0);
8312 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
8324 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
8328 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF);
8336 MVT VT = Op.getSimpleValueType();
8340 if (VT.isVector()) {
8341 MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
8343 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
8351 VT.isScalarInteger()) {
8354 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV);
8357 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV);
8363 ISD::OR, DL, VT, TrueV,
8364 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
8369 ISD::OR, DL, VT, FalseV,
8370 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV));
8387 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
8389 DAG.getConstant(IsCZERO_NEZ ? TrueVal : FalseVal, DL, VT);
8392 DL, VT, LHSVal, CondV);
8393 return DAG.getNode(ISD::ADD, DL, VT, CMOV, RHSVal);
8400 ISD::OR, DL, VT,
8401 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
8402 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
8430 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV);
8434 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR);
8449 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
8473 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
8475 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
8482 LHS = DAG.getConstant(0, DL, VT);
8493 RHS = DAG.getConstant(0, DL, VT);
8506 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
8556 EVT VT = Op.getValueType();
8558 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
8563 ISD::ADD, DL, VT, FrameAddr,
8566 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
8583 EVT VT = Op.getValueType();
8589 SDValue Offset = DAG.getSignedConstant(Off, DL, VT);
8590 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
8591 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
8607 EVT VT = Lo.getValueType();
8616 SDValue Zero = DAG.getConstant(0, DL, VT);
8617 SDValue One = DAG.getConstant(1, DL, VT);
8618 SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
8619 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
8620 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
8621 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
8623 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
8624 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
8626 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
8627 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
8628 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
8629 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
8631 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
8633 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
8634 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
8646 EVT VT = Lo.getValueType();
8666 SDValue Zero = DAG.getConstant(0, DL, VT);
8667 SDValue One = DAG.getConstant(1, DL, VT);
8668 SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
8669 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
8670 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
8671 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
8673 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
8674 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
8676 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
8677 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
8678 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
8679 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
8681 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
8683 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
8685 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
8686 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
8697 MVT VT = Op.getSimpleValueType();
8701 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
8702 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
8705 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
8706 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
8708 MVT InterVT = VT.changeVectorElementType(MVT::i8);
8713 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
8794 MVT VT = Op.getOperand(0).getSimpleValueType();
8801 MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
8808 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
8876 MVT VT = Op.getSimpleValueType();
8878 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
8881 if (VT.getVectorElementType() == MVT::i1)
8887 MVT DstEltVT = VT.getVectorElementType();
8933 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
8944 MVT VT = Op.getSimpleValueType();
8946 MVT ContainerVT = VT;
8947 if (VT.isFixedLengthVector()) {
8950 SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
8957 if ((VT.getVectorElementType() == MVT::f64 &&
8960 ((VT.getVectorElementType() == MVT::f16 ||
8961 VT.getVectorElementType() == MVT::bf16) &&
8978 if (VT.isFixedLengthVector()) {
8981 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
8998 MVT VT = Op.getSimpleValueType();
9000 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
9006 IsExtend && (VT.getVectorElementType() != MVT::f64 ||
9009 bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
9010 VT.getVectorElementType() != MVT::bf16) ||
9016 MVT ContainerVT = VT;
9022 if (VT.isFixedLengthVector()) {
9025 SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
9041 if (VT.isFixedLengthVector())
9042 Src = convertFromScalableVector(VT, Src, DAG, Subtarget);
9054 if (VT.isFixedLengthVector())
9055 return convertFromScalableVector(VT, Result, DAG, Subtarget);
9475 // Use the previous operand to get the vXi64 VT. The result might be a mask
9476 // VT for compares. Using the previous operand assumes that the previous
9480 // element count from any operand or result and use it to construct the VT.
9482 MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
9486 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
9505 MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
9518 RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget);
9533 RISCVII::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT);
9535 unsigned Sew = RISCVVType::encodeSEW(VT.getScalarSizeInBits());
9571 Vec = DAG.getBitcast(VT, Vec);
9585 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
9586 DAG.getUNDEF(VT), AVL);
9589 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
9597 ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG);
9728 static inline bool isValidEGW(int EGS, EVT VT,
9731 VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
9732 EGS * VT.getScalarSizeInBits();
9843 assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
9860 MVT VT = Op.getSimpleValueType();
9864 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG);
9868 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
9871 MVT MaskVT = getMaskTypeFor(VT);
9872 SDValue Mask = getAllOnesMask(VT, VL, DL, DAG);
9873 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
9878 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal,
9879 Vec, DAG.getUNDEF(VT), VL);
9940 MVT VT = Op.getSimpleValueType();
9945 MVT RetVT = VT;
9946 if (VT.isFixedLengthVector())
9947 RetVT = getContainerForFixedLengthVector(VT);
9948 else if (VT.isFloatingPoint())
9949 RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()),
9950 VT.getVectorElementCount());
9954 if (VT.isFixedLengthVector())
9955 NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget);
9956 else if (VT.isFloatingPoint())
9957 NewNode = DAG.getBitcast(VT, NewNode);
9977 MVT VT = Op.getSimpleValueType();
9978 MVT RetVT = VT;
9979 MVT FloatVT = VT;
9981 if (VT.isFloatingPoint()) {
9982 RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()),
9983 VT.getVectorElementCount());
9986 if (VT.isFixedLengthVector())
9996 if (VT.isFixedLengthVector())
9998 if (VT.isFloatingPoint())
9999 NewNode = DAG.getBitcast(VT, NewNode);
10037 MVT VT = Op->getSimpleValueType(0);
10038 MVT ContainerVT = getContainerForFixedLengthVector(VT);
10043 SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
10054 DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
10063 Results.push_back(convertFromScalableVector(VT, SubVec, DAG, Subtarget));
10124 MVT VT = Op->getOperand(2).getSimpleValueType();
10125 MVT ContainerVT = getContainerForFixedLengthVector(VT);
10130 SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
10150 DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
10327 // or the original VT if fractional.
10946 MVT VT = N.getSimpleValueType();
10947 MVT WideVT = VT.changeVectorElementType(MVT::i8);
10950 assert(Op.getSimpleValueType() == VT &&
10984 // If the VT is LMUL=8, we need to split and reassemble.
11066 // If the VT is LMUL=8, we need to split and reassemble.
11145 MVT VT = Op.getSimpleValueType();
11146 assert(VT.isScalableVector() && "Expected scalable vector");
11148 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
11149 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
11154 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
11156 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
11159 SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()),
11160 VL, VT, DL, DAG, Subtarget);
11161 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
11354 MVT VT = Op.getSimpleValueType();
11356 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11362 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
11369 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
11373 SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
11375 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
11388 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
11404 MVT VT = StoreVal.getSimpleValueType();
11408 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
11409 VT = MVT::v8i1;
11411 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getConstant(0, DL, VT),
11415 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11424 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
11432 SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
11434 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
11446 MVT VT = Op.getSimpleValueType();
11458 PassThru = DAG.getUNDEF(VT);
11471 MVT ContainerVT = VT;
11472 if (VT.isFixedLengthVector()) {
11473 ContainerVT = getContainerForFixedLengthVector(VT);
11482 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11521 if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
11538 if (VT.isFixedLengthVector())
11539 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
11570 MVT VT = Val.getSimpleValueType();
11573 MVT ContainerVT = VT;
11574 if (VT.isFixedLengthVector()) {
11575 ContainerVT = getContainerForFixedLengthVector(VT);
11585 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11616 MVT VT = Val.getSimpleValueType();
11618 MVT ContainerVT = VT;
11619 if (VT.isFixedLengthVector()) {
11620 ContainerVT = getContainerForFixedLengthVector(VT);
11627 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11633 if (VT.isFixedLengthVector())
11634 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
11645 MVT VT = Op.getSimpleValueType();
11653 auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL,
11661 return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
11673 MVT VT = Op.getSimpleValueType();
11692 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
11701 SDValue Res = DAG.getNOT(DL, OEQ, VT);
11745 if (VT.isFixedLengthVector()) {
11746 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
11755 MVT VT = Op.getSimpleValueType();
11758 assert((Op.getOpcode() == ISD::VP_ABS || VT.isFixedLengthVector()) &&
11761 MVT ContainerVT = VT;
11762 if (VT.isFixedLengthVector()) {
11763 ContainerVT = getContainerForFixedLengthVector(VT);
11770 if (VT.isFixedLengthVector())
11775 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11785 if (VT.isFixedLengthVector())
11786 Max = convertFromScalableVector(VT, Max, DAG, Subtarget);
11793 MVT VT = Op.getSimpleValueType();
11799 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11803 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11808 return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
11813 MVT VT = Op.getSimpleValueType();
11814 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11827 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11832 return convertFromScalableVector(VT, Select, DAG, Subtarget);
11841 MVT VT = Op.getSimpleValueType();
11842 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11862 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11875 SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
11881 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
11894 MVT VT = Op.getSimpleValueType();
11897 MVT ContainerVT = VT;
11898 if (VT.isFixedLengthVector())
11899 ContainerVT = getContainerForFixedLengthVector(VT);
11941 if (!VT.isFixedLengthVector())
11942 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
11946 return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
11952 MVT VT = Op.getSimpleValueType();
11958 MVT ContainerVT = VT;
11959 if (VT.isFixedLengthVector()) {
11960 ContainerVT = getContainerForFixedLengthVector(VT);
11977 if (!VT.isFixedLengthVector())
11979 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11985 MVT VT = Op.getSimpleValueType();
11993 MVT ContainerVT = VT;
11994 if (VT.isFixedLengthVector()) {
11995 ContainerVT = getContainerForFixedLengthVector(VT);
12056 if (!VT.isFixedLengthVector())
12058 return convertFromScalableVector(VT, Result, DAG, Subtarget);
12189 MVT VT = Op.getSimpleValueType();
12190 if (!VT.isFixedLengthVector())
12192 return convertFromScalableVector(VT, Result, DAG, Subtarget);
12198 MVT VT = Op.getSimpleValueType();
12208 VT.getVectorElementCount());
12212 MVT ContainerVT = VT;
12213 if (VT.isFixedLengthVector()) {
12214 ContainerVT = getContainerForFixedLengthVector(VT);
12249 if (VT.isFixedLengthVector())
12250 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
12267 MVT VT = Op.getSimpleValueType();
12268 MVT ContainerVT = VT;
12269 if (VT.isFixedLengthVector()) {
12270 ContainerVT = getContainerForFixedLengthVector(VT);
12277 bool IsMaskVector = VT.getVectorElementType() == MVT::i1;
12330 if (!VT.isFixedLengthVector())
12332 return convertFromScalableVector(VT, Result, DAG, Subtarget);
12341 MVT VT = Op.getSimpleValueType();
12343 MVT ContainerVT = VT;
12344 if (VT.isFixedLengthVector()) {
12345 ContainerVT = getContainerForFixedLengthVector(VT);
12353 if (!VT.isFixedLengthVector())
12355 return convertFromScalableVector(VT, Result, DAG, Subtarget);
12362 MVT VT = Op.getSimpleValueType();
12369 MVT ContainerVT = VT;
12370 if (VT.isFixedLengthVector()) {
12371 ContainerVT = getContainerForFixedLengthVector(VT);
12446 if (!VT.isFixedLengthVector())
12448 return convertFromScalableVector(VT, Result, DAG, Subtarget);
12474 if (!VT.isFixedLengthVector())
12476 return convertFromScalableVector(VT, Result, DAG, Subtarget);
12481 MVT VT = Op.getSimpleValueType();
12482 if (VT.getVectorElementType() != MVT::i1)
12490 MVT ContainerVT = VT;
12491 const bool IsFixed = VT.isFixedLengthVector();
12493 ContainerVT = getContainerForFixedLengthVector(VT);
12502 return convertFromScalableVector(VT, Val, DAG, Subtarget);
12509 MVT VT = Op.getSimpleValueType();
12510 MVT ContainerVT = VT;
12511 if (VT.isFixedLengthVector())
12512 ContainerVT = getContainerForFixedLengthVector(VT);
12528 if (VT.isFixedLengthVector()) {
12545 if (VT.isFixedLengthVector())
12546 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
12558 MVT VT = StoreVal.getSimpleValueType();
12559 MVT ContainerVT = VT;
12560 if (VT.isFixedLengthVector()) {
12561 ContainerVT = getContainerForFixedLengthVector(VT);
12575 if (VT.isFixedLengthVector()) {
12597 MVT VT = Op.getSimpleValueType();
12611 PassThru = DAG.getUNDEF(VT);
12627 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
12638 MVT ContainerVT = VT;
12639 if (VT.isFixedLengthVector()) {
12640 ContainerVT = getContainerForFixedLengthVector(VT);
12654 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
12681 if (VT.isFixedLengthVector())
12682 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
12721 MVT VT = Val.getSimpleValueType();
12725 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
12736 MVT ContainerVT = VT;
12737 if (VT.isFixedLengthVector()) {
12738 ContainerVT = getContainerForFixedLengthVector(VT);
12752 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
13149 MVT VT = N->getSimpleValueType(0);
13150 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
13164 if (VT != MVT::i32)
13297 EVT VT = N->getValueType(0);
13298 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
13302 if (VT == MVT::i16 &&
13307 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
13312 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() &&
13319 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
13325 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
13328 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
13336 MVT VT = N->getSimpleValueType(0);
13338 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
13347 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes));
13533 EVT VT = N->getValueType(0);
13535 if (VT.bitsLT(XLenVT)) {
13539 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
13543 assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
13653 const EVT VT = N->getValueType(0);
13660 if (!VT.isInteger() &&
13683 assert(SrcVecVT.getVectorElementType() == VT);
13691 // reduce_op (extract_subvector [2 x VT] from V). This will form the
13699 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, 2);
13702 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags());
13721 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, RHSIdx + 1);
13724 return DAG.getNode(ReduceOpc, DL, VT, Vec,
13846 EVT VT = N->getValueType(0);
13847 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
13877 SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, NL,
13878 DAG.getConstant(Diff, DL, VT), NS);
13879 return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
13897 EVT VT = N->getValueType(0);
13900 if (VT.isVector())
13914 // Maybe harmful when VT is wider than XLen.
13915 if (VT.getSizeInBits() > Subtarget.getXLen())
13944 FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
13950 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
13954 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
13992 EVT VT = N->getValueType(0);
13993 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
14031 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
14032 DAG.getSignedConstant(CA, DL, VT));
14034 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getSignedConstant(C0, DL, VT));
14035 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getSignedConstant(CB, DL, VT));
14051 EVT VT = N->getValueType(0);
14052 if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
14067 SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
14071 EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C);
14072 EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount());
14086 OuterExtend, SDLoc(N), VT,
14094 EVT VT = N->getValueType(0);
14106 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
14111 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
14142 EVT VT = N->getValueType(0);
14166 DAG.getSetCC(SDLoc(N1), VT, N1.getOperand(0), N1.getOperand(1), CCVal);
14175 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT);
14176 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
14188 EVT VT = N->getValueType(0);
14190 if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
14226 APInt Mask = APInt::getSplat(VT.getSizeInBits(), APInt(8, 0x1));
14233 return DAG.getNode(RISCVISD::ORC_B, SDLoc(N), VT, LeftShiftOperand);
14241 EVT VT = N->getValueType(0);
14251 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
14252 DAG.getConstant(ShAmt, DL, VT));
14296 EVT VT = N->getValueType(0);
14302 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
14309 SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10);
14310 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
14320 EVT VT = N->getValueType(0);
14325 if (!VT.isVector() || !TLI.isTypeLegal(VT) || !TLI.isTypeLegal(SrcVT))
14350 unsigned ScalarBits = VT.getScalarSizeInBits();
14380 return DAG.getNode(ISD::TRUNCATE, DL, VT, Min);
14386 EVT VT = N->getValueType(0);
14393 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
14400 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl);
14472 EVT VT = N->getValueType(0);
14475 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0),
14477 SDValue NewN1 = DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0),
14479 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1);
14480 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1));
14546 EVT VT = N0.getValueType();
14550 return DAG.getSetCC(DL, VT, N0.getOperand(1),
14551 DAG.getConstant(Imm + 1, DL, VT), CC);
14570 EVT VT = N->getValueType(0);
14579 if (VT != Subtarget.getXLenVT())
14609 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
14610 DAG.getConstant(Log2_64(MulAmt2), DL, VT));
14611 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Shl,
14612 DAG.getConstant(Log2_64(Divisor - 1), DL, VT),
14618 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14619 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
14620 return DAG.getNode(ISD::SHL, DL, VT, Mul359,
14621 DAG.getConstant(Log2_64(MulAmt2), DL, VT));
14628 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14629 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
14630 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Mul359,
14631 DAG.getConstant(Log2_64(MulAmt2 - 1), DL, VT),
14645 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
14646 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14647 DAG.getConstant(ScaleShift, DL, VT), Shift1);
14664 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14665 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
14666 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Mul359,
14667 DAG.getConstant(TZ, DL, VT), X);
14678 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
14679 return DAG.getNode(ISD::ADD, DL, VT, Shift1,
14680 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14681 DAG.getConstant(ScaleShift, DL, VT), X));
14690 DAG.getNode(ISD::SHL, DL, VT, X,
14691 DAG.getConstant(Log2_64(MulAmt + Offset), DL, VT));
14693 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14694 DAG.getConstant(Log2_64(Offset - 1), DL, VT), X);
14695 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Mul359);
14705 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14706 DAG.getConstant(Log2_64(ShiftAmt1), DL, VT));
14708 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14709 DAG.getConstant(Log2_64(MulAmtLowBit), DL, VT));
14710 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Shift2);
14727 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
14728 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
14730 RISCVISD::SHL_ADD, DL, VT, Mul359A,
14731 DAG.getConstant(Log2_64(Divisor2 - 1), DL, VT), Mul359A);
14732 return DAG.getNode(ISD::SHL, DL, VT, Mul359B,
14733 DAG.getConstant(Log2_64(MulAmt3), DL, VT));
14746 EVT VT = N->getValueType(0);
14750 if (!VT.isVector() || !TLI.isTypeLegal(VT) || VT.getScalarSizeInBits() < 16)
14766 unsigned HalfSize = VT.getScalarSizeInBits() / 2;
14773 VT.getVectorElementCount() * 2);
14778 return DAG.getNode(ISD::BITCAST, DL, VT, Sra);
14784 EVT VT = N->getValueType(0);
14785 if (!VT.isVector())
14812 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
14813 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal);
14817 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
14818 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
14839 EVT VT = N.getValueType();
14854 if (ResultVT.bitsLT(VT.getVectorElementType())) {
14856 VT.changeVectorElementType(ResultVT), N);
14905 EVT VT = N->getValueType(0);
14939 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
14943 return DAG.getSetCC(dl, VT, SExtOp, DAG.getConstant(C1.trunc(32).sext(64),
14951 EVT VT = N->getValueType(0);
14959 return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, SDLoc(N), VT,
14964 VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
14966 return DAG.getNode(RISCVISD::SLLW, SDLoc(N), VT, Src.getOperand(0),
15114 MVT VT = Root->getSimpleValueType(0);
15117 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
15125 MVT NarrowVT = MVT::getVectorVT(EltVT, VT.getVectorElementCount());
15244 MVT VT = OrigOperand.getSimpleValueType();
15259 unsigned EltBits = VT.getScalarSizeInBits();
15321 MVT VT = OrigOperand.getSimpleValueType();
15322 if (!VT.isVector())
15354 MVT VT = OrigOperand.getSimpleValueType();
15367 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
15476 MVT VT = Root->getSimpleValueType(0);
15477 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
16104 EVT VT = N->getValueType(0);
16106 if (VT.isVector() && TLI.isTypeLegal(VT)) {
16109 MVT ContainerVT = VT.getSimpleVT();
16116 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits() * 2 ||
16117 VT.getScalarSizeInBits() * 2 < SrcVT.getScalarSizeInBits())
16146 if (VT.isFixedLengthVector())
16147 FpToInt = convertFromScalableVector(VT, FpToInt, DAG, Subtarget);
16154 if (VT != MVT::i32 && VT != XLenVT)
16158 if (VT == XLenVT)
16165 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
16246 EVT VT = N->getValueType(0);
16247 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() ||
16248 !llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
16252 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0));
16475 EVT VT = N->getValueType(0);
16477 if (VT != Subtarget.getXLenVT())
16496 unsigned Size = VT.getSizeInBits();
16499 DAG.getNode(ISD::SHL, ShlDL, VT, N0.getOperand(0).getOperand(0),
16500 DAG.getConstant(LShAmt + (Size - ExtSize), ShlDL, VT));
16502 return DAG.getNode(ISD::SRA, DL, VT, Shl,
16503 DAG.getConstant(ShAmt + (Size - ExtSize), DL, VT));
16508 if (ShAmt > 32 || VT != MVT::i64)
16612 EVT VT = Cond.getValueType();
16616 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
16628 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(0),
16632 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(1),
16633 DAG.getConstant(1, SDLoc(Setcc), VT), CCVal);
16636 Setcc = DAG.getSetCC(SDLoc(Setcc), VT,
16637 DAG.getConstant(0, SDLoc(Setcc), VT),
16643 return DAG.getNode(Opc, SDLoc(Cond), VT, Setcc, Xor.getOperand(0));
16771 EVT VT = N->getValueType(0);
16785 return DAG.getNode(TrueVal.getOpcode(), DL, VT, FalseVal, NewSel);
16855 EVT VT = N->getValueType(0);
16863 if (Subtarget.hasStdExtZbs() && VT.isScalarInteger() &&
16872 return DAG.getSelect(DL, VT,
16905 EVT VT = N->getValueType(0);
16920 SDValue NegB = DAG.getNegative(B, DL, VT);
16923 SDValue NewB = DAG.getNode(ISD::VSELECT, DL, VT, CC, NegB, B);
16924 return DAG.getNode(ISD::ADD, DL, VT, A, NewB);
16954 EVT VT = N->getValueType(0);
16956 assert(!VT.isScalableVector() && "unexpected build vector");
16958 if (VT.getVectorNumElements() == 1)
16965 if (!TLI.isOperationLegalOrCustom(Opcode, VT) || !TLI.isTypeLegal(VT))
16970 if (N->op_begin()->getValueType() != VT.getVectorElementType())
17004 return DAG.getNode(Opcode, DL, VT, DAG.getBuildVector(VT, DL, LHSOps),
17005 DAG.getBuildVector(VT, DL, RHSOps));
17016 EVT VT = InVec.getValueType();
17017 if (VT.isScalableVector())
17042 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
17044 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
17046 return DAG.getNode(InVecOpcode, DL, VT, LHS, RHS);
17073 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
17084 EVT VT = N->getValueType(0);
17087 if (!TLI.isTypeLegal(VT))
17091 if (VT.isScalableVector())
17203 return DAG.getBitcast(VT.getSimpleVT(), StridedLoad);
17210 EVT VT = N->getValueType(0);
17211 const unsigned ElementSize = VT.getScalarSizeInBits();
17212 const unsigned NumElts = VT.getVectorNumElements();
17237 SDValue NegB = DAG.getNegative(B, DL, VT);
17240 SDValue NewB = DAG.getNode(ISD::VSELECT, DL, VT, CC, NegB, B);
17241 return DAG.getNode(ISD::ADD, DL, VT, A, NewB);
17248 if (TLI.isTypeLegal(VT) || ElementSize <= Subtarget.getELen() ||
17249 !isPowerOf2_64(ElementSize) || VT.getVectorNumElements() % 2 != 0 ||
17250 VT.isFloatingPoint() || TLI.isShuffleMaskLegal(Mask, VT))
17258 EVT NewVT = EVT::getVectorVT(C, NewEltVT, VT.getVectorNumElements() * 2);
17261 return DAG.getBitcast(VT, Res);
17330 EVT VT = N->getValueType(0);
17333 return DAG.getNode(Opc, DL, VT, Ops);
17369 static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask,
17376 const unsigned ElementSize = VT.getScalarStoreSize();
17377 const unsigned NumElems = VT.getVectorNumElements();
17404 static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask,
17413 const unsigned NumElems = VT.getVectorNumElements();
17417 const unsigned ElementSize = VT.getScalarStoreSize();
17509 MVT VT = N->getSimpleValueType(0);
17554 if (HiC.isMask(VT.getScalarSizeInBits()))
17562 if (LoC.isNonNegative() && HiC.isMask(VT.getScalarSizeInBits()))
17571 if (LoC.isNonNegative() && HiC.isMask(VT.getScalarSizeInBits()) &&
17581 unsigned NumDstBits = VT.getScalarSizeInBits();
17625 } while (ValVT != VT);
17639 EVT VT = N->getValueType(0);
17640 if (!VT.isScalarInteger())
17673 return DAG.getZExtOrTrunc(Pop, DL, VT);
17777 MVT VT = N->getSimpleValueType(0);
17781 APInt Val = CFP->getValueAPF().bitcastToAPInt().sext(VT.getSizeInBits());
17782 return DAG.getConstant(Val, DL, VT);
17792 assert(Op0.getOperand(0).getValueType() == VT &&
17802 DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, LN0->getChain(),
17815 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
17817 APInt SignBit = APInt::getSignMask(FPBits).sext(VT.getSizeInBits());
17819 return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
17820 DAG.getConstant(SignBit, DL, VT));
17823 return DAG.getNode(ISD::AND, DL, VT, NewFMV,
17824 DAG.getConstant(~SignBit, DL, VT));
17827 EVT VT = N->getValueType(0);
17831 if (VT.isVector() && N0.hasOneUse() && N0.getOpcode() == ISD::SIGN_EXTEND) {
17834 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
17882 EVT VT = N->getValueType(0);
17883 if (VT.isVector() || !isOperationLegal(ISD::FCOPYSIGN, VT))
17886 if (Sign.getValueType() != VT)
17888 return DAG.getNode(RISCVISD::FSGNJX, SDLoc(N), VT, N1, N0->getOperand(1));
17982 EVT VT = N->getValueType(0);
18003 DAG.getNode(ISD::SRA, DL, VT, LHS,
18004 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
18006 DAG.getNode(ISD::AND, DL, VT, SRA,
18007 DAG.getSignedConstant(TrueSImm - FalseSImm, DL, VT));
18008 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
18022 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal);
18023 SDValue Neg = DAG.getNegative(C, DL, VT);
18024 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV);
18029 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
18030 SDValue Neg = DAG.getNegative(C, DL, VT);
18031 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV);
18037 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
18038 SDValue Neg = DAG.getNegative(C, DL, VT);
18039 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV);
18043 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal);
18044 SDValue Neg = DAG.getNegative(C, DL, VT);
18045 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV);
18056 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ);
18057 return DAG.getNode(ISD::ADD, DL, VT, LHS, C);
18069 SDValue NewSel = DAG.getNode(RISCVISD::SELECT_CC, DL, VT, LHS, RHS, CC,
18071 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1));
18097 EVT VT = N->getValueType(0);
18098 if (!VT.isVector())
18116 SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
18117 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
18118 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
18122 const EVT VT = N->getValueType(0);
18145 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) {
18165 VT.getVectorElementCount());
18167 VT, DL, MGN->getChain(), BasePtr,
18170 SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
18179 matchIndexAsShuffle(VT, Index, MGN->getMask(), ShuffleMask)) {
18180 SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(),
18182 MGN->getMask(), DAG.getUNDEF(VT),
18186 DAG.getVectorShuffle(VT, DL, Load, DAG.getUNDEF(VT), ShuffleMask);
18191 matchIndexAsWiderOp(VT, Index, MGN->getMask(),
18200 unsigned ElementSize = VT.getScalarStoreSize();
18202 auto EltCnt = VT.getVectorElementCount();
18216 SDValue Result = DAG.getBitcast(VT, Gather.getValue(0));
18244 EVT VT = MSN->getValue()->getValueType(0);
18247 matchIndexAsShuffle(VT, Index, MSN->getMask(), ShuffleMask)) {
18248 SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(),
18249 DAG.getUNDEF(VT), ShuffleMask);
18317 EVT VT = N->getValueType(0);
18318 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
18320 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
18339 EVT VT = N->getValueType(0);
18340 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
18343 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
18461 // VecVT should be scalable and memory VT should match the element type.
18478 EVT VT = N->getValueType(0);
18480 if (!isTypeLegal(VT))
18482 if (auto Gather = matchSplatAsGather(N->getOperand(0), VT.getSimpleVT(), N,
18504 const MVT VT = N->getSimpleValueType(0);
18511 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL);
18515 const MVT VT = N->getSimpleValueType(0);
18523 unsigned EltWidth = VT.getScalarSizeInBits();
18534 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL);
18547 EVT VT = N->getValueType(0);
18549 assert(SrcVT.getVectorElementType() == VT.getVectorElementType());
18551 if (SrcVT == VT)
18558 const MVT VT = N->getSimpleValueType(0);
18568 const MVT M1VT = getLMUL1VT(VT);
18569 if (M1VT.bitsLT(VT)) {
18575 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result,
18585 VT.bitsLE(getLMUL1VT(VT)) && Passthru.isUndef())
18586 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
18622 EVT VT = N->getValueType(0);
18625 return DAG.getAllOnesConstant(DL, VT);
18626 return DAG.getConstant(0, DL, VT);
18637 EVT VT = N->getValueType(0);
18639 if (VT.isRISCVVectorTuple() && N0->getOpcode() == ISD::SPLAT_VECTOR) {
18640 unsigned NF = VT.getRISCVVectorTupleNumFields();
18641 unsigned NumScalElts = VT.getSizeInBits().getKnownMinValue() / (NF * 8);
18647 SDValue Result = DAG.getUNDEF(VT);
18649 Result = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VT, Result, Splat,
18656 VT.isScalarInteger()) {
18663 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
18791 EVT VT = Op.getValueType();
18792 if (VT.isVector())
18839 if (VT == MVT::i64) {
19953 MVT ArgVT = Ins[i].VT;
19978 MVT ArgVT = Outs[i].VT;
20376 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
20378 VT.getTypeForEVT(*DAG.getContext()));
20710 MVT VT = Outs[i].VT;
20712 if (CC_RISCV(i, VT, VT, CCValAssign::Full, ArgFlags, CCInfo,
21178 MVT VT) const {
21185 if (VT.isVector())
21187 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
21189 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
21191 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
21195 if (VT == MVT::f16) {
21200 } else if (VT == MVT::f32) {
21205 } else if (VT == MVT::f64) {
21228 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
21241 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
21245 if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
21248 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
21250 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
21252 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
21254 if (!VT.isVector())
21259 if (VT == MVT::f16) {
21264 } else if (VT == MVT::f32) {
21269 } else if (VT == MVT::f64) {
21364 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
21369 if (VT == MVT::f32 || VT == MVT::Other)
21371 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
21415 if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
21417 if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
21421 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
21430 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
21717 EVT VT) const {
21718 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
21803 EVT VT;
21806 VT = LD->getMemoryVT();
21809 VT = ST->getMemoryVT();
21846 EVT VT;
21849 VT = LD->getMemoryVT();
21852 VT = ST->getMemoryVT();
21869 EVT VT) const {
21870 EVT SVT = VT.getScalarType();
21877 return VT.isVector() ? Subtarget.hasVInstructionsF16()
21923 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
21926 if (!VT.isScalarInteger())
21932 if (HasZmmul && VT.getSizeInBits() > Subtarget.getXLen())
21965 EVT VT = AddNode.getValueType();
21966 if (VT.isVector())
21970 if (VT.getScalarSizeInBits() > Subtarget.getXLen())
21986 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
21988 if (!VT.isVector()) {
21995 EVT ElemVT = VT.getVectorElementType();
22046 // Do we have sufficient alignment for our preferred VT? If not, revert
22220 bool RISCVTargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
22225 return OptSize && !VT.isVector();
22271 EVT VT = getValueType(DL, VTy);
22273 if (!isTypeLegal(VT))
22276 if (!isLegalElementTypeForRVV(VT.getScalarType()) ||
22277 !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace,
22281 MVT ContainerVT = VT.getSimpleVT();
22291 ContainerVT = getContainerForFixedLengthVector(VT.getSimpleVT());
22633 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
22698 bool RISCVTargetLowering::isCtpopFast(EVT VT) const {
22699 if (VT.isScalableVector())
22700 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
22701 if (VT.isFixedLengthVector() && Subtarget.hasStdExtZvbb())
22704 (VT == MVT::i32 || VT == MVT::i64 || VT.isFixedLengthVector());
22707 unsigned RISCVTargetLowering::getCustomCtpopCost(EVT VT,
22709 return isCtpopFast(VT) ? 0 : 1;
22779 EVT VT = N->getValueType(0);
22780 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit())))
22790 EVT VT, const APInt &AndMask) const {
22793 return TargetLowering::shouldFoldSelectWithSingleBitTest(VT, AndMask);
22818 // VT of that MachineSDNode. This VT may be used to look up the representative
22830 MVT VT) const {
22831 switch (VT.SimpleTy) {
22843 return TargetLowering::findRepresentativeClass(TRI, VT);
22889 EVT VT = Op.getValueType();
22896 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
22897 DAG.getSignedConstant(-(uint64_t)Align->value(), dl, VT));