Lines Matching defs:VT
171 static inline EVT getPackedSVEVectorVT(EVT VT) {
172 switch (VT.getSimpleVT().SimpleTy) {
211 static inline EVT getPromotedVTForPredicate(EVT VT) {
212 assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) &&
214 switch (VT.getVectorMinNumElements()) {
228 /// Returns true if VT's elements occupy the lowest bit positions of its
233 static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
234 assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
236 return VT.isFixedLengthVector() ||
237 VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock;
453 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
454 if (useSVEForFixedLengthVectorVT(VT))
455 addRegisterClass(VT, &AArch64::ZPRRegClass);
457 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
458 if (useSVEForFixedLengthVectorVT(VT))
459 addRegisterClass(VT, &AArch64::ZPRRegClass);
633 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
634 setOperationAction(ISD::ROTL, VT, Expand);
635 setOperationAction(ISD::ROTR, VT, Expand);
685 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
686 setOperationAction(ISD::SDIVREM, VT, Expand);
687 setOperationAction(ISD::UDIVREM, VT, Expand);
1039 for (MVT VT : MVT::fp_valuetypes()) {
1040 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
1041 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
1042 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1043 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
1044 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
1046 for (MVT VT : MVT::integer_valuetypes())
1047 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
1240 for (auto VT : {MVT::v2i32, MVT::v2i64, MVT::v4i32})
1241 setOperationAction(Op, VT, Custom);
1276 for (auto VT : {MVT::v1i64, MVT::v2i64}) {
1277 setOperationAction(ISD::UMAX, VT, Custom);
1278 setOperationAction(ISD::SMAX, VT, Custom);
1279 setOperationAction(ISD::UMIN, VT, Custom);
1280 setOperationAction(ISD::SMIN, VT, Custom);
1292 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1294 setOperationAction(ISD::SADDSAT, VT, Legal);
1295 setOperationAction(ISD::UADDSAT, VT, Legal);
1296 setOperationAction(ISD::SSUBSAT, VT, Legal);
1297 setOperationAction(ISD::USUBSAT, VT, Legal);
1300 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1302 setOperationAction(ISD::AVGFLOORS, VT, Legal);
1303 setOperationAction(ISD::AVGFLOORU, VT, Legal);
1304 setOperationAction(ISD::AVGCEILS, VT, Legal);
1305 setOperationAction(ISD::AVGCEILU, VT, Legal);
1306 setOperationAction(ISD::ABDS, VT, Legal);
1307 setOperationAction(ISD::ABDU, VT, Legal);
1311 for (MVT VT : { MVT::v4f16, MVT::v2f32,
1313 if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
1314 setOperationAction(ISD::VECREDUCE_FMAX, VT, Legal);
1315 setOperationAction(ISD::VECREDUCE_FMIN, VT, Legal);
1316 setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Legal);
1317 setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Legal);
1319 setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
1322 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1324 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1325 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1326 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1327 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1328 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1329 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1330 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1331 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1342 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1343 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
1345 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
1346 setOperationAction(ISD::MULHS, VT, Legal);
1347 setOperationAction(ISD::MULHU, VT, Legal);
1349 setOperationAction(ISD::MULHS, VT, Expand);
1350 setOperationAction(ISD::MULHU, VT, Expand);
1352 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1353 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1355 setOperationAction(ISD::BSWAP, VT, Expand);
1356 setOperationAction(ISD::CTTZ, VT, Expand);
1359 setTruncStoreAction(VT, InnerVT, Expand);
1360 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1361 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1362 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1407 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1408 setOperationAction(ISD::ADD, VT, Custom);
1410 for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
1411 setOperationAction(ISD::FADD, VT, Custom);
1413 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1415 setOperationAction(Op, VT, Expand);
1417 if (VT.is128BitVector() || VT.is64BitVector()) {
1418 setOperationAction(ISD::LOAD, VT, Legal);
1419 setOperationAction(ISD::STORE, VT, Legal);
1420 setOperationAction(ISD::BITCAST, VT,
1424 setTruncStoreAction(VT, InnerVT, Expand);
1425 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1426 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1427 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1432 for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
1433 setOperationAction(ISD::TRUNCATE_SSAT_S, VT, Legal);
1434 setOperationAction(ISD::TRUNCATE_SSAT_U, VT, Legal);
1435 setOperationAction(ISD::TRUNCATE_USAT_U, VT, Legal);
1445 for (auto VT :
1447 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1448 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1449 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1450 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1455 for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
1456 setOperationAction(ISD::BITREVERSE, VT, Custom);
1457 setOperationAction(ISD::BSWAP, VT, Custom);
1458 setOperationAction(ISD::CTLZ, VT, Custom);
1459 setOperationAction(ISD::CTPOP, VT, Custom);
1460 setOperationAction(ISD::CTTZ, VT, Custom);
1461 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1462 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1463 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1464 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1465 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1466 setOperationAction(ISD::MLOAD, VT, Custom);
1467 setOperationAction(ISD::MUL, VT, Custom);
1468 setOperationAction(ISD::MULHS, VT, Custom);
1469 setOperationAction(ISD::MULHU, VT, Custom);
1470 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1471 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
1472 setOperationAction(ISD::SELECT, VT, Custom);
1473 setOperationAction(ISD::SETCC, VT, Custom);
1474 setOperationAction(ISD::SDIV, VT, Custom);
1475 setOperationAction(ISD::UDIV, VT, Custom);
1476 setOperationAction(ISD::SMIN, VT, Custom);
1477 setOperationAction(ISD::UMIN, VT, Custom);
1478 setOperationAction(ISD::SMAX, VT, Custom);
1479 setOperationAction(ISD::UMAX, VT, Custom);
1480 setOperationAction(ISD::SHL, VT, Custom);
1481 setOperationAction(ISD::SRL, VT, Custom);
1482 setOperationAction(ISD::SRA, VT, Custom);
1483 setOperationAction(ISD::ABS, VT, Custom);
1484 setOperationAction(ISD::ABDS, VT, Custom);
1485 setOperationAction(ISD::ABDU, VT, Custom);
1486 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1487 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1488 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1489 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1490 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1491 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1492 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1493 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1494 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1495 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1497 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1498 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1499 setOperationAction(ISD::SELECT_CC, VT, Expand);
1500 setOperationAction(ISD::ROTL, VT, Expand);
1501 setOperationAction(ISD::ROTR, VT, Expand);
1503 setOperationAction(ISD::SADDSAT, VT, Legal);
1504 setOperationAction(ISD::UADDSAT, VT, Legal);
1505 setOperationAction(ISD::SSUBSAT, VT, Legal);
1506 setOperationAction(ISD::USUBSAT, VT, Legal);
1507 setOperationAction(ISD::UREM, VT, Expand);
1508 setOperationAction(ISD::SREM, VT, Expand);
1509 setOperationAction(ISD::SDIVREM, VT, Expand);
1510 setOperationAction(ISD::UDIVREM, VT, Expand);
1512 setOperationAction(ISD::AVGFLOORS, VT, Custom);
1513 setOperationAction(ISD::AVGFLOORU, VT, Custom);
1514 setOperationAction(ISD::AVGCEILS, VT, Custom);
1515 setOperationAction(ISD::AVGCEILU, VT, Custom);
1518 setOperationAction(ISD::BITCAST, VT, Custom);
1523 setOperationAction(ISD::OR, VT, Custom);
1527 for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
1528 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1529 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1533 for (auto VT : {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32})
1534 setOperationAction(ISD::BITCAST, VT, Custom);
1536 for (auto VT :
1539 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
1541 for (auto VT :
1543 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1544 setOperationAction(ISD::SELECT, VT, Custom);
1545 setOperationAction(ISD::SETCC, VT, Custom);
1546 setOperationAction(ISD::TRUNCATE, VT, Custom);
1547 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1548 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1549 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1551 setOperationAction(ISD::SELECT_CC, VT, Expand);
1552 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1553 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1556 if (VT != MVT::nxv16i1) {
1557 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1558 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1563 for (auto VT :
1567 setOperationAction(ISD::MLOAD, VT, Custom);
1568 setOperationAction(ISD::MSTORE, VT, Custom);
1573 for (MVT VT : MVT::scalable_vector_valuetypes()) {
1575 setTruncStoreAction(VT, InnerVT, Expand);
1576 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1577 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1578 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1605 for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1607 setOperationAction(ISD::BITCAST, VT, Custom);
1608 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1609 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1610 setOperationAction(ISD::MLOAD, VT, Custom);
1611 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1612 setOperationAction(ISD::SELECT, VT, Custom);
1613 setOperationAction(ISD::SETCC, VT, Custom);
1614 setOperationAction(ISD::FADD, VT, Custom);
1615 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1616 setOperationAction(ISD::FDIV, VT, Custom);
1617 setOperationAction(ISD::FMA, VT, Custom);
1618 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1619 setOperationAction(ISD::FMAXNUM, VT, Custom);
1620 setOperationAction(ISD::FMINIMUM, VT, Custom);
1621 setOperationAction(ISD::FMINNUM, VT, Custom);
1622 setOperationAction(ISD::FMUL, VT, Custom);
1623 setOperationAction(ISD::FNEG, VT, Custom);
1624 setOperationAction(ISD::FSUB, VT, Custom);
1625 setOperationAction(ISD::FCEIL, VT, Custom);
1626 setOperationAction(ISD::FFLOOR, VT, Custom);
1627 setOperationAction(ISD::FNEARBYINT, VT, Custom);
1628 setOperationAction(ISD::FRINT, VT, Custom);
1629 setOperationAction(ISD::LRINT, VT, Custom);
1630 setOperationAction(ISD::LLRINT, VT, Custom);
1631 setOperationAction(ISD::FROUND, VT, Custom);
1632 setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1633 setOperationAction(ISD::FTRUNC, VT, Custom);
1634 setOperationAction(ISD::FSQRT, VT, Custom);
1635 setOperationAction(ISD::FABS, VT, Custom);
1636 setOperationAction(ISD::FP_EXTEND, VT, Custom);
1637 setOperationAction(ISD::FP_ROUND, VT, Custom);
1638 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1639 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1640 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1641 setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Custom);
1642 setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Custom);
1643 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
1644 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1645 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1647 setOperationAction(ISD::SELECT_CC, VT, Expand);
1648 setOperationAction(ISD::FREM, VT, Expand);
1649 setOperationAction(ISD::FPOW, VT, Expand);
1650 setOperationAction(ISD::FPOWI, VT, Expand);
1651 setOperationAction(ISD::FCOS, VT, Expand);
1652 setOperationAction(ISD::FSIN, VT, Expand);
1653 setOperationAction(ISD::FSINCOS, VT, Expand);
1654 setOperationAction(ISD::FTAN, VT, Expand);
1655 setOperationAction(ISD::FACOS, VT, Expand);
1656 setOperationAction(ISD::FASIN, VT, Expand);
1657 setOperationAction(ISD::FATAN, VT, Expand);
1658 setOperationAction(ISD::FATAN2, VT, Expand);
1659 setOperationAction(ISD::FCOSH, VT, Expand);
1660 setOperationAction(ISD::FSINH, VT, Expand);
1661 setOperationAction(ISD::FTANH, VT, Expand);
1662 setOperationAction(ISD::FEXP, VT, Expand);
1663 setOperationAction(ISD::FEXP2, VT, Expand);
1664 setOperationAction(ISD::FEXP10, VT, Expand);
1665 setOperationAction(ISD::FLOG, VT, Expand);
1666 setOperationAction(ISD::FLOG2, VT, Expand);
1667 setOperationAction(ISD::FLOG10, VT, Expand);
1669 setCondCodeAction(ISD::SETO, VT, Expand);
1670 setCondCodeAction(ISD::SETOLT, VT, Expand);
1671 setCondCodeAction(ISD::SETLT, VT, Expand);
1672 setCondCodeAction(ISD::SETOLE, VT, Expand);
1673 setCondCodeAction(ISD::SETLE, VT, Expand);
1674 setCondCodeAction(ISD::SETULT, VT, Expand);
1675 setCondCodeAction(ISD::SETULE, VT, Expand);
1676 setCondCodeAction(ISD::SETUGE, VT, Expand);
1677 setCondCodeAction(ISD::SETUGT, VT, Expand);
1678 setCondCodeAction(ISD::SETUEQ, VT, Expand);
1679 setCondCodeAction(ISD::SETONE, VT, Expand);
1682 for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
1683 setOperationAction(ISD::BITCAST, VT, Custom);
1684 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1685 setOperationAction(ISD::FABS, VT, Legal);
1686 setOperationAction(ISD::FNEG, VT, Legal);
1687 setOperationAction(ISD::FP_EXTEND, VT, Custom);
1688 setOperationAction(ISD::FP_ROUND, VT, Custom);
1689 setOperationAction(ISD::MLOAD, VT, Custom);
1690 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1691 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1692 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
1695 setOperationAction(ISD::FADD, VT, Legal);
1696 setOperationAction(ISD::FMA, VT, Custom);
1697 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1698 setOperationAction(ISD::FMAXNUM, VT, Custom);
1699 setOperationAction(ISD::FMINIMUM, VT, Custom);
1700 setOperationAction(ISD::FMINNUM, VT, Custom);
1701 setOperationAction(ISD::FMUL, VT, Legal);
1702 setOperationAction(ISD::FSUB, VT, Legal);
1727 for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
1729 setOperationAction(ISD::SDIV, VT, Custom);
1730 setOperationAction(ISD::UDIV, VT, Custom);
1740 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1742 VT, /*OverrideNEON=*/!Subtarget->isNeonAvailable()))
1743 addTypeForFixedLengthSVE(VT);
1745 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1747 VT, /*OverrideNEON=*/!Subtarget->isNeonAvailable()))
1748 addTypeForFixedLengthSVE(VT);
1752 for (auto VT : {MVT::v8i8, MVT::v4i16})
1753 setOperationAction(ISD::TRUNCATE, VT, Custom);
1757 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
1758 setOperationAction(ISD::TRUNCATE, VT, Custom);
1759 for (auto VT : {MVT::v8f16, MVT::v4f32})
1760 setOperationAction(ISD::FP_ROUND, VT, Custom);
1785 for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
1787 setOperationAction(ISD::BITREVERSE, VT, Custom);
1788 setOperationAction(ISD::CTTZ, VT, Custom);
1789 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1790 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1791 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1792 setOperationAction(ISD::MULHS, VT, Custom);
1793 setOperationAction(ISD::MULHU, VT, Custom);
1797 for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
1798 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1808 for (auto VT : {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1})
1809 setOperationAction(ISD::INTRINSIC_WO_CHAIN, VT, Custom);
1814 for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64,
1821 setOperationAction(ISD::MGATHER, VT, Custom);
1822 setOperationAction(ISD::MSCATTER, VT, Custom);
1825 for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1828 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1831 for (auto VT :
1834 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1838 for (auto VT : {MVT::v2i8, MVT::v2i16, MVT::v2i32, MVT::v2i64, MVT::v2f32,
1840 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1897 void AArch64TargetLowering::addTypeForNEON(MVT VT) {
1898 assert(VT.isVector() && "VT should be a vector type");
1900 if (VT.isFloatingPoint()) {
1901 MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
1902 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
1903 setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
1907 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
1908 setOperationAction(ISD::FSIN, VT, Expand);
1909 setOperationAction(ISD::FCOS, VT, Expand);
1910 setOperationAction(ISD::FTAN, VT, Expand);
1911 setOperationAction(ISD::FASIN, VT, Expand);
1912 setOperationAction(ISD::FACOS, VT, Expand);
1913 setOperationAction(ISD::FATAN, VT, Expand);
1914 setOperationAction(ISD::FATAN2, VT, Expand);
1915 setOperationAction(ISD::FSINH, VT, Expand);
1916 setOperationAction(ISD::FCOSH, VT, Expand);
1917 setOperationAction(ISD::FTANH, VT, Expand);
1918 setOperationAction(ISD::FPOW, VT, Expand);
1919 setOperationAction(ISD::FLOG, VT, Expand);
1920 setOperationAction(ISD::FLOG2, VT, Expand);
1921 setOperationAction(ISD::FLOG10, VT, Expand);
1922 setOperationAction(ISD::FEXP, VT, Expand);
1923 setOperationAction(ISD::FEXP2, VT, Expand);
1924 setOperationAction(ISD::FEXP10, VT, Expand);
1928 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
1929 ((VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v4f16 ||
1930 VT == MVT::v8f16) &&
1932 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1934 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1935 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1936 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1937 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1938 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1939 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1940 setOperationAction(ISD::SRA, VT, Custom);
1941 setOperationAction(ISD::SRL, VT, Custom);
1942 setOperationAction(ISD::SHL, VT, Custom);
1943 setOperationAction(ISD::OR, VT, Custom);
1944 setOperationAction(ISD::SETCC, VT, Custom);
1945 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
1947 setOperationAction(ISD::SELECT, VT, Expand);
1948 setOperationAction(ISD::SELECT_CC, VT, Expand);
1949 setOperationAction(ISD::VSELECT, VT, Expand);
1951 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1954 if (VT != MVT::v8i8 && VT != MVT::v16i8)
1955 setOperationAction(ISD::CTPOP, VT, Custom);
1957 setOperationAction(ISD::UDIV, VT, Expand);
1958 setOperationAction(ISD::SDIV, VT, Expand);
1959 setOperationAction(ISD::UREM, VT, Expand);
1960 setOperationAction(ISD::SREM, VT, Expand);
1961 setOperationAction(ISD::FREM, VT, Expand);
1966 setOperationAction(Opcode, VT, Custom);
1968 if (!VT.isFloatingPoint())
1969 setOperationAction(ISD::ABS, VT, Legal);
1972 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
1974 setOperationAction(Opcode, VT, Legal);
1978 if (VT.isFloatingPoint() &&
1979 VT.getVectorElementType() != MVT::bf16 &&
1980 (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
1987 setOperationAction(Opcode, VT, Legal);
1990 if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 16)
1991 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
1992 if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 64)
1993 setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
2006 setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
2007 setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
2012 setIndexedLoadAction(im, VT, Legal);
2013 setIndexedStoreAction(im, VT, Legal);
2048 EVT VT = EVT::getEVT(I->getType());
2051 if (Op1VT.getVectorElementType() == VT.getVectorElementType() &&
2052 (VT.getVectorElementCount() * 4 == Op1VT.getVectorElementCount() ||
2053 VT.getVectorElementCount() * 2 == Op1VT.getVectorElementCount()))
2058 bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
2064 return VT != MVT::nxv16i1 && VT != MVT::nxv8i1 && VT != MVT::nxv4i1 &&
2065 VT != MVT::nxv2i1 && VT != MVT::v16i1 && VT != MVT::v8i1 &&
2066 VT != MVT::v4i1 && VT != MVT::v2i1;
2069 bool AArch64TargetLowering::shouldExpandVectorMatch(EVT VT,
2075 if (VT == MVT::nxv8i16 || VT == MVT::v8i16)
2077 if (VT == MVT::nxv16i8 || VT == MVT::v16i8 || VT == MVT::v8i8)
2082 void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
2083 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
2087 setOperationAction(Op, VT, Expand);
2089 if (VT.isFloatingPoint()) {
2090 setCondCodeAction(ISD::SETO, VT, Expand);
2091 setCondCodeAction(ISD::SETOLT, VT, Expand);
2092 setCondCodeAction(ISD::SETOLE, VT, Expand);
2093 setCondCodeAction(ISD::SETULT, VT, Expand);
2094 setCondCodeAction(ISD::SETULE, VT, Expand);
2095 setCondCodeAction(ISD::SETUGE, VT, Expand);
2096 setCondCodeAction(ISD::SETUGT, VT, Expand);
2097 setCondCodeAction(ISD::SETUEQ, VT, Expand);
2098 setCondCodeAction(ISD::SETONE, VT, Expand);
2102 VT == MVT::v1f64 ? Expand : Custom;
2105 if (VT.isInteger()) {
2106 MVT InnerVT = VT.changeVectorElementType(MVT::i8);
2107 while (InnerVT != VT) {
2108 setTruncStoreAction(VT, InnerVT, Default);
2109 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Default);
2110 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Default);
2111 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Default);
2119 if (VT.isFloatingPoint()) {
2120 MVT InnerVT = VT.changeVectorElementType(MVT::f16);
2121 while (InnerVT != VT) {
2122 setTruncStoreAction(VT, InnerVT, Custom);
2123 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Default);
2129 bool PreferNEON = VT.is64BitVector() || VT.is128BitVector();
2133 setOperationAction(ISD::ABDS, VT, Default);
2134 setOperationAction(ISD::ABDU, VT, Default);
2135 setOperationAction(ISD::ABS, VT, Default);
2136 setOperationAction(ISD::ADD, VT, Default);
2137 setOperationAction(ISD::AND, VT, Default);
2138 setOperationAction(ISD::ANY_EXTEND, VT, Default);
2139 setOperationAction(ISD::BITCAST, VT, PreferNEON ? Legal : Default);
2140 setOperationAction(ISD::BITREVERSE, VT, Default);
2141 setOperationAction(ISD::BSWAP, VT, Default);
2142 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2143 setOperationAction(ISD::CONCAT_VECTORS, VT, Default);
2144 setOperationAction(ISD::CTLZ, VT, Default);
2145 setOperationAction(ISD::CTPOP, VT, Default);
2146 setOperationAction(ISD::CTTZ, VT, Default);
2147 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Default);
2148 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Default);
2149 setOperationAction(ISD::FABS, VT, Default);
2150 setOperationAction(ISD::FADD, VT, Default);
2151 setOperationAction(ISD::FCEIL, VT, Default);
2152 setOperationAction(ISD::FCOPYSIGN, VT, Default);
2153 setOperationAction(ISD::FDIV, VT, Default);
2154 setOperationAction(ISD::FFLOOR, VT, Default);
2155 setOperationAction(ISD::FMA, VT, Default);
2156 setOperationAction(ISD::FMAXIMUM, VT, Default);
2157 setOperationAction(ISD::FMAXNUM, VT, Default);
2158 setOperationAction(ISD::FMINIMUM, VT, Default);
2159 setOperationAction(ISD::FMINNUM, VT, Default);
2160 setOperationAction(ISD::FMUL, VT, Default);
2161 setOperationAction(ISD::FNEARBYINT, VT, Default);
2162 setOperationAction(ISD::FNEG, VT, Default);
2163 setOperationAction(ISD::FP_EXTEND, VT, Default);
2164 setOperationAction(ISD::FP_ROUND, VT, Default);
2165 setOperationAction(ISD::FP_TO_SINT, VT, Default);
2166 setOperationAction(ISD::FP_TO_UINT, VT, Default);
2167 setOperationAction(ISD::FRINT, VT, Default);
2168 setOperationAction(ISD::LRINT, VT, Default);
2169 setOperationAction(ISD::LLRINT, VT, Default);
2170 setOperationAction(ISD::FROUND, VT, Default);
2171 setOperationAction(ISD::FROUNDEVEN, VT, Default);
2172 setOperationAction(ISD::FSQRT, VT, Default);
2173 setOperationAction(ISD::FSUB, VT, Default);
2174 setOperationAction(ISD::FTRUNC, VT, Default);
2175 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Default);
2176 setOperationAction(ISD::LOAD, VT, PreferNEON ? Legal : Default);
2177 setOperationAction(ISD::MGATHER, VT, PreferSVE ? Default : Expand);
2178 setOperationAction(ISD::MLOAD, VT, Default);
2179 setOperationAction(ISD::MSCATTER, VT, PreferSVE ? Default : Expand);
2180 setOperationAction(ISD::MSTORE, VT, Default);
2181 setOperationAction(ISD::MUL, VT, Default);
2182 setOperationAction(ISD::MULHS, VT, Default);
2183 setOperationAction(ISD::MULHU, VT, Default);
2184 setOperationAction(ISD::OR, VT, Default);
2185 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, PreferNEON ? Legal : Expand);
2186 setOperationAction(ISD::SDIV, VT, Default);
2187 setOperationAction(ISD::SELECT, VT, Default);
2188 setOperationAction(ISD::SETCC, VT, Default);
2189 setOperationAction(ISD::SHL, VT, Default);
2190 setOperationAction(ISD::SIGN_EXTEND, VT, Default);
2191 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Default);
2192 setOperationAction(ISD::SINT_TO_FP, VT, Default);
2193 setOperationAction(ISD::SMAX, VT, Default);
2194 setOperationAction(ISD::SMIN, VT, Default);
2195 setOperationAction(ISD::SPLAT_VECTOR, VT, Default);
2196 setOperationAction(ISD::SRA, VT, Default);
2197 setOperationAction(ISD::SRL, VT, Default);
2198 setOperationAction(ISD::STORE, VT, PreferNEON ? Legal : Default);
2199 setOperationAction(ISD::SUB, VT, Default);
2200 setOperationAction(ISD::TRUNCATE, VT, Default);
2201 setOperationAction(ISD::UDIV, VT, Default);
2202 setOperationAction(ISD::UINT_TO_FP, VT, Default);
2203 setOperationAction(ISD::UMAX, VT, Default);
2204 setOperationAction(ISD::UMIN, VT, Default);
2205 setOperationAction(ISD::VECREDUCE_ADD, VT, Default);
2206 setOperationAction(ISD::VECREDUCE_AND, VT, Default);
2207 setOperationAction(ISD::VECREDUCE_FADD, VT, Default);
2208 setOperationAction(ISD::VECREDUCE_FMAX, VT, Default);
2209 setOperationAction(ISD::VECREDUCE_FMIN, VT, Default);
2210 setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Default);
2211 setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Default);
2212 setOperationAction(ISD::VECREDUCE_OR, VT, Default);
2213 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, PreferSVE ? Default : Expand);
2214 setOperationAction(ISD::VECREDUCE_SMAX, VT, Default);
2215 setOperationAction(ISD::VECREDUCE_SMIN, VT, Default);
2216 setOperationAction(ISD::VECREDUCE_UMAX, VT, Default);
2217 setOperationAction(ISD::VECREDUCE_UMIN, VT, Default);
2218 setOperationAction(ISD::VECREDUCE_XOR, VT, Default);
2219 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Default);
2220 setOperationAction(ISD::VECTOR_SPLICE, VT, Default);
2221 setOperationAction(ISD::VSELECT, VT, Default);
2222 setOperationAction(ISD::XOR, VT, Default);
2223 setOperationAction(ISD::ZERO_EXTEND, VT, Default);
2226 void AArch64TargetLowering::addDRType(MVT VT) {
2227 addRegisterClass(VT, &AArch64::FPR64RegClass);
2229 addTypeForNEON(VT);
2232 void AArch64TargetLowering::addQRType(MVT VT) {
2233 addRegisterClass(VT, &AArch64::FPR128RegClass);
2235 addTypeForNEON(VT);
2239 LLVMContext &C, EVT VT) const {
2240 if (!VT.isVector())
2242 if (VT.isScalableVector())
2243 return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
2244 return VT.changeVectorElementTypeToInteger();
2340 EVT VT = Op.getValueType();
2347 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
2348 TLO.DAG.getConstant(NewImm, DL, VT));
2353 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
2355 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
2371 EVT VT = Op.getValueType();
2372 if (VT.isVector())
2375 unsigned Size = VT.getSizeInBits();
2486 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
2487 unsigned MemBits = VT.getScalarSizeInBits();
2501 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
2503 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
2504 unsigned Bound = (VT == MVT::v8i8) ? 11 : 12;
2517 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
2519 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
2523 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
2538 EVT VT = Op.getValueType();
2539 unsigned VTBits = VT.getScalarSizeInBits();
2578 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2587 if (VT.isScalableVector()) {
2588 unsigned ElementSizeBits = VT.getScalarSizeInBits();
2598 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
2609 VT == MVT::v2i64;
3413 static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT);
3414 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3415 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3418 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
3420 EVT VT);
3625 EVT VT = LHS.getValueType();
3626 assert(VT != MVT::f128);
3630 if ((VT == MVT::f16 && !FullFP16) || VT == MVT::bf16) {
3644 EVT VT = LHS.getValueType();
3647 if (VT.isFloatingPoint()) {
3648 assert(VT != MVT::f128);
3649 if ((VT == MVT::f16 && !FullFP16) || VT == MVT::bf16) {
3678 DAG.getVTList(VT, MVT_CC),
3690 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
4017 EVT VT = Op.getValueType();
4018 if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
4029 EVT VT = RHS.getValueType();
4038 if ((VT == MVT::i32 && C != 0x80000000 &&
4040 (VT == MVT::i64 && C != 0x80000000ULL &&
4043 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
4044 RHS = DAG.getConstant(C, dl, VT);
4049 if ((VT == MVT::i32 && C != 0 &&
4051 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
4053 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
4054 RHS = DAG.getConstant(C, dl, VT);
4059 if ((VT == MVT::i32 && C != INT32_MAX &&
4061 (VT == MVT::i64 && C != INT64_MAX &&
4064 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
4065 RHS = DAG.getConstant(C, dl, VT);
4070 if ((VT == MVT::i32 && C != UINT32_MAX &&
4072 (VT == MVT::i64 && C != UINT64_MAX &&
4075 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
4076 RHS = DAG.getConstant(C, dl, VT);
4339 EVT VT = Value.getValueType();
4340 SDValue Op0 = Invert ? DAG.getConstant(0, DL, VT) : Value;
4341 SDValue Op1 = Invert ? Value : DAG.getConstant(1, DL, VT);
4343 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::Glue), Op0, Op1);
4349 static SDValue carryFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG,
4353 SDValue Zero = DAG.getConstant(0, DL, VT);
4354 SDValue One = DAG.getConstant(1, DL, VT);
4357 return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Glue);
4361 static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG) {
4364 SDValue Zero = DAG.getConstant(0, DL, VT);
4365 SDValue One = DAG.getConstant(1, DL, VT);
4367 return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Glue);
4481 EVT VT = Op.getValueType();
4482 if (VT.isScalableVector()) {
4489 if (VT == MVT::nxv2f32 || VT == MVT::nxv4f32)
4492 if (VT != MVT::nxv2f64)
4496 // and the second using native f32->VT instructions.
4498 return DAG.getNode(ISD::FP_EXTEND, DL, VT,
4505 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
4511 if (VT == MVT::f64) {
4520 return DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(Op), {VT, MVT::Other},
4524 return DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), VT,
4529 if (VT.getScalarType() == MVT::f32) {
4535 EVT IVT = VT.changeTypeToInteger();
4549 Shift = DAG.getBitcast(VT, Shift);
4562 EVT VT = Op.getValueType();
4568 if (VT.isScalableVector()) {
4569 if (VT.getScalarType() != MVT::bf16)
4601 return DAG.getNode(Op.getOpcode(), DL, VT, NewOps, Op->getFlags());
4616 EVT CondVT = VT.changeElementType(MVT::i1);
4624 return getSVESafeBitCast(VT, Narrow, DAG);
4632 if (VT.getScalarType() == MVT::bf16 &&
4676 if (VT.isVector()) {
4679 return DAG.getNode(ISD::BITCAST, dl, VT, Narrow);
4682 SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow);
4706 EVT VT = Op.getValueType();
4708 if (VT.isScalableVector()) {
4715 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()) ||
4729 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
4737 uint64_t VTSize = VT.getFixedSizeInBits();
4745 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
4751 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
4757 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
4758 VT.getVectorNumElements());
4762 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
4766 return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
4776 EVT ScalarVT = VT.getScalarType();
4983 EVT VT = Op.getValueType();
4987 assert(VT.isVector() && "Expected vector type");
4990 VT.changeVectorElementType(Src.getValueType().getVectorElementType());
4997 return DAG.getNode(ISD::FP_TO_SINT_SAT, DL, VT, FOp,
4998 DAG.getValueType(VT.getVectorElementType()));
5007 EVT VT = Op.getValueType();
5014 if (VT.isScalableVector()) {
5020 return DAG.getNode(Opc, dl, VT, In);
5028 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()) ||
5033 if (VT.getVectorElementType() == MVT::bf16) {
5034 EVT F32 = VT.changeElementType(MVT::f32);
5048 uint64_t VTSize = VT.getFixedSizeInBits();
5057 return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other},
5062 return DAG.getNode(ISD::FP_ROUND, dl, VT, In,
5068 EVT CastVT = VT.changeVectorElementTypeToInteger();
5071 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op.getOperand(0), In});
5072 return DAG.getNode(Opc, dl, VT, In);
5077 if (VT.getVectorNumElements() == 1) {
5081 EVT ScalarVT = VT.getScalarType();
5332 EVT VT = OpNode->getOperand(0).getValueType();
5334 if (!VT.isFixedLengthVector() || VT.getVectorNumElements() != 2 || !C)
5342 EVT VT = N.getValueType();
5349 unsigned EltSize = VT.getScalarSizeInBits();
5367 EVT VT = N.getValueType();
5368 assert(VT.is128BitVector() && "Unexpected vector MULL size");
5371 VT.getScalarType().getHalfSizedIntegerVT(*DAG.getContext()),
5372 VT.getVectorElementCount());
5550 EVT VT = N0.getValueType();
5551 unsigned EltSize = VT.getScalarSizeInBits();
5556 } else if (VT == MVT::v2i64 && DAG.MaskedValueIsZero(N0, Mask) &&
5566 } else if (VT == MVT::v2i64 && DAG.ComputeNumSignBits(N0) > EltSize / 2 &&
5593 EVT VT = Op.getValueType();
5596 if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
5601 assert((VT.is128BitVector() || VT.is64BitVector()) && VT.isInteger() &&
5606 EVT OVT = VT;
5607 if (VT.is64BitVector()) {
5614 VT = N0.getValueType();
5616 if (VT == MVT::v1i64) {
5631 if (VT.getVectorElementType() == MVT::i64) {
5652 DAG.getNode(NewOpc, DL, VT, Op0, Op1),
5663 DAG.getNode(N0.getOpcode(), DL, VT,
5664 DAG.getNode(NewOpc, DL, VT,
5666 DAG.getNode(NewOpc, DL, VT,
5671 static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
5673 if (VT == MVT::nxv1i1 && Pattern == AArch64SVEPredPattern::all)
5675 return DAG.getNode(AArch64ISD::PTRUE, DL, VT,
5724 static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
5729 VT.getVectorElementType() == MVT::i1 &&
5731 assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
5737 if (InVT == VT)
5741 // than VT. This will increase the chances of removing casts that introduce
5745 Op.getOperand(1).getValueType().bitsGT(VT))
5748 SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
5754 if (InVT.bitsGT(VT))
5764 Mask = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Mask);
5765 return DAG.getNode(ISD::AND, DL, VT, Reinterpret, Mask);
5770 EVT VT) const {
6484 EVT VT = Op.getValueType();
6485 if (VT.isScalableVector())
6486 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, ID, Op.getOperand(1),
6493 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
6499 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, MaskAsInt,
6524 EVT VT = CttzOp.getValueType();
6525 assert(VT.getVectorElementType() == MVT::i1 && "Expected MVT::i1");
6527 if (VT.isFixedLengthVector()) {
6530 EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
6545 bool AArch64TargetLowering::shouldExtendGSIndex(EVT VT, EVT &EltTy) const {
6546 if (VT.getVectorElementType() == MVT::i8 ||
6547 VT.getVectorElementType() == MVT::i16) {
6656 EVT VT = Op.getValueType();
6664 SDValue Ops[] = {Chain, DAG.getUNDEF(VT), Mask, BasePtr, Index, Scale};
6668 SDValue Select = DAG.getSelect(DL, VT, Mask, Load, PassThru);
6691 if (VT.isFixedLengthVector()) {
6696 EVT DataVT = VT.changeVectorElementTypeToInteger();
6700 EVT PromotedVT = VT.changeVectorElementType(MVT::i32);
6704 PromotedVT = VT.changeVectorElementType(MVT::i64);
6734 if (VT.isFloatingPoint())
6735 Result = DAG.getNode(ISD::BITCAST, DL, VT, Result);
6755 EVT VT = StoreVal.getValueType();
6779 if (VT.isFixedLengthVector()) {
6784 if (VT.isFloatingPoint()) {
6785 VT = VT.changeVectorElementTypeToInteger();
6787 StoreVal = DAG.getNode(ISD::BITCAST, DL, VT, StoreVal);
6791 EVT PromotedVT = VT.changeVectorElementType(MVT::i32);
6792 if (VT.getVectorElementType() == MVT::i64 ||
6795 PromotedVT = VT.changeVectorElementType(MVT::i64);
6804 if (PromotedVT != VT)
6829 EVT VT = Op->getValueType(0);
6831 if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
6841 VT, DL, LoadNode->getChain(), LoadNode->getBasePtr(),
6842 LoadNode->getOffset(), Mask, DAG.getUNDEF(VT), LoadNode->getMemoryVT(),
6846 SDValue Result = DAG.getSelect(DL, VT, Mask, Load, PassThru);
6853 EVT VT, EVT MemVT,
6855 assert(VT.isVector() && "VT should be a vector type");
6856 assert(MemVT == MVT::v4i8 && VT == MVT::v4i16);
6894 EVT VT = Value.getValueType();
6897 if (VT.isVector()) {
6899 VT,
6912 if (StoreNode->isTruncatingStore() && VT == MVT::v4i16 &&
6914 return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG);
7020 EVT VT = Op->getValueType(0);
7021 assert((VT == MVT::v4i16 || VT == MVT::v4i32) && "Expected v4i16 or v4i32");
7047 if (VT == MVT::v4i32)
7152 MVT VT = Op.getSimpleValueType();
7154 if (VT.isVector())
7158 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
7162 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
7163 Op.getOperand(0), DAG.getConstant(0, DL, VT));
7164 return DAG.getNode(AArch64ISD::CSEL, DL, VT, Op.getOperand(0), Neg,
7193 MVT VT = Op.getSimpleValueType();
7197 VT.getFixedSizeInBits() - ShiftNo->getZExtValue();
7199 ISD::FSHR, DL, VT, Op.getOperand(0), Op.getOperand(1),
7659 bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const {
7664 EVT VT, bool OverrideNEON) const {
7665 if (!VT.isFixedLengthVector() || !VT.isSimple())
7669 switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
7686 if (OverrideNEON && (VT.is128BitVector() || VT.is64BitVector()))
7690 if (VT.getFixedSizeInBits() <= 128)
7698 if (VT.getFixedSizeInBits() > Subtarget->getMinSVEVectorSizeInBits())
7703 if (!VT.isPow2VectorType())
7820 static bool isPassedInFPR(EVT VT) {
7821 return VT.isFixedLengthVector() ||
7822 (VT.isFloatingPoint() && !VT.isScalableVector());
7841 if (any_of(Outs, [](ISD::OutputArg &Out){ return Out.VT.isScalableVector(); }))
7849 // At this point, Ins[].VT may already be promoted to i32. To correctly
7852 // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
7859 MVT ValVT = Ins[i].VT;
8513 MVT ArgVT = Outs[i].VT;
8973 if (!Outs[i].IsFixed && Outs[i].VT.isScalableVector())
9180 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
9299 Outs[0].VT == MVT::i64) {
9302 assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
10403 SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC,
10432 EVT VT = Op.getValueType();
10465 SDValue TPtr = DAG.getTargetGlobalAddress(PtrGV, DL, VT, PtrOffsetC,
10495 TPtr, DL, VT, (AArch64PACKey::ID)KeyC, Discriminator, AddrDiscriminator,
10658 EVT VT = Op.getValueType();
10659 EVT IntVT = VT.changeTypeToInteger();
10666 if (!SrcVT.bitsEq(VT))
10667 In2 = DAG.getFPExtendOrRound(In2, DL, VT);
10669 if (VT.isScalableVector())
10671 getPackedSVEVectorVT(VT.getVectorElementType().changeTypeToInteger());
10673 if (VT.isFixedLengthVector() &&
10674 useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable())) {
10675 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
10681 return convertFromScalableVector(DAG, VT, Res);
10684 auto BitCast = [this](EVT VT, SDValue Op, SelectionDAG &DAG) {
10685 if (VT.isScalableVector())
10686 return getSVESafeBitCast(VT, Op, DAG);
10688 return DAG.getBitcast(VT, Op);
10694 if (!VT.isVector()) {
10704 if (VT.isVector()) {
10707 } else if (VT == MVT::f64) {
10710 } else if (VT == MVT::f32) {
10713 } else if (VT == MVT::f16 || VT == MVT::bf16) {
10726 if (VT == MVT::f64 || VT == MVT::v2f64) {
10735 if (VT == MVT::f16 || VT == MVT::bf16)
10736 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, BSP);
10737 if (VT == MVT::f32)
10738 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, BSP);
10739 if (VT == MVT::f64)
10740 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, BSP);
10742 return BitCast(VT, BSP, DAG);
10751 EVT VT = Op.getValueType();
10752 if (VT.isScalableVector() ||
10753 useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
10765 if (VT == MVT::i32 && IsParity)
10776 if (VT == MVT::i32 || VT == MVT::i64) {
10777 if (VT == MVT::i32)
10783 if (VT == MVT::i32)
10786 AddV = DAG.getNode(ISD::BITCAST, DL, VT, AddV);
10788 AddV = DAG.getNode(ISD::AND, DL, VT, AddV, DAG.getConstant(1, DL, VT));
10790 } else if (VT == MVT::i128) {
10795 AddV = DAG.getNode(ISD::BITCAST, DL, VT, AddV);
10797 AddV = DAG.getNode(ISD::AND, DL, VT, AddV, DAG.getConstant(1, DL, VT));
10803 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
10804 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
10807 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
10811 if (Subtarget->hasDotProd() && VT.getScalarSizeInBits() != 16 &&
10812 VT.getVectorNumElements() >= 2) {
10813 EVT DT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
10817 if (VT == MVT::v2i64) {
10819 Val = DAG.getNode(AArch64ISD::UADDLP, DL, VT, Val);
10820 } else if (VT == MVT::v2i32) {
10822 } else if (VT == MVT::v4i32) {
10831 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
10833 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
10834 while (EltSize != VT.getScalarSizeInBits()) {
10845 EVT VT = Op.getValueType();
10846 assert(VT.isScalableVector() ||
10848 VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()));
10851 SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0));
10852 return DAG.getNode(ISD::CTLZ, DL, VT, RBIT);
10858 EVT VT = Op.getValueType();
10879 if (VT.isScalableVector() ||
10881 VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors())) {
10898 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
10899 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
10904 EVT VT = Op.getValueType();
10906 if (VT.isScalableVector() ||
10908 VT, /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()))
10915 switch (VT.getSimpleVT().SimpleTy) {
10948 return DAG.getNode(AArch64ISD::NVCAST, DL, VT,
10985 EVT VT = N->getValueType(0);
11002 SDValue Cmp = DAG.getSetCC(DL, VT, XOR0, XOR1, Cond);
11005 SDValue CmpChain = DAG.getSetCC(DL, VT, XOR0, XOR1, Cond);
11006 Cmp = DAG.getNode(LogicOp, DL, VT, Cmp, CmpChain);
11033 EVT VT = Op.getValueType();
11034 SDValue TVal = DAG.getConstant(1, dl, VT);
11035 SDValue FVal = DAG.getConstant(0, dl, VT);
11062 SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
11089 Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
11099 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
11102 Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
11112 EVT VT = LHS.getValueType();
11113 if (VT != MVT::i32 && VT != MVT::i64)
11120 SDValue Cmp = DAG.getNode(AArch64ISD::SBCS, DL, DAG.getVTList(VT, MVT::Glue),
11128 ISD::CondCode CondInv = ISD::getSetCCInverse(Cond, VT);
11175 EVT VT = LHS.getValueType();
11177 DAG.getNode(ISD::SRA, dl, VT, LHS,
11178 DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
11179 return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT));
11189 EVT VT = LHS.getValueType();
11191 DAG.getNode(ISD::SRA, dl, VT, LHS,
11192 DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
11195 Shift = DAG.getNOT(dl, Shift, VT);
11197 return DAG.getNode(ISD::AND, dl, VT, LHS, Shift);
11314 EVT VT = TVal.getValueType();
11315 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
11322 EVT VT = TVal.getValueType();
11350 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
11356 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
11792 EVT VT = Op.getValueType();
11805 if (VT.isScalableVector())
11816 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11823 if (VT.isInteger() && !VT.isVector())
11826 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
11847 DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
11854 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
11862 EVT VT = Op.getValueType();
11868 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
11873 DAG.getValueType(VT));
11882 EVT VT = getPointerTy(DAG.getDataLayout());
11885 return DAG.getFrameIndex(FI, VT);
11894 getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const {
11913 EVT VT = Op.getValueType();
11917 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
11920 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset);
11929 EVT VT = Op.getValueType();
11937 VT, DL, DAG.getEntryNode(),
11938 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), MachinePointerInfo());
11943 ReturnAddress = DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
11952 St = DAG.getMachineNode(AArch64::XPACI, DL, VT, ReturnAddress);
11957 St = DAG.getMachineNode(AArch64::XPACLRI, DL, VT, Chain);
11978 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
11990 if (VT == MVT::f64)
11992 else if (VT == MVT::f32)
11994 else if (VT == MVT::f16 || VT == MVT::bf16)
12004 if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) {
12011 AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(), Insn);
12018 LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT
12030 EVT VT = Operand.getValueType();
12032 (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 ||
12033 VT == MVT::f32 || VT == MVT::v1f32 || VT == MVT::v2f32 ||
12034 VT == MVT::v4f32)) ||
12036 (VT == MVT::nxv8f16 || VT == MVT::nxv4f32 || VT == MVT::nxv2f64))) {
12044 unsigned DesiredBits = APFloat::semanticsPrecision(VT.getFltSemantics());
12050 return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
12060 EVT VT = Op.getValueType();
12061 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
12062 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
12082 EVT VT = Operand.getValueType();
12089 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
12091 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
12092 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
12095 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
12111 EVT VT = Operand.getValueType();
12118 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
12120 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
12221 getPredicateRegisterClass(PredicateConstraint Constraint, EVT VT) {
12222 if (VT != MVT::aarch64svcount &&
12223 (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1))
12228 return VT == MVT::aarch64svcount ? &AArch64::PNR_p8to15RegClass
12231 return VT == MVT::aarch64svcount ? &AArch64::PNR_3bRegClass
12234 return VT == MVT::aarch64svcount ? &AArch64::PNRRegClass
12252 getReducedGprRegisterClass(ReducedGprConstraint Constraint, EVT VT) {
12253 if (!VT.isScalarInteger() || VT.getFixedSizeInBits() > 64)
12408 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
12412 if (VT.isScalableVector())
12414 if (Subtarget->hasLS64() && VT.getSizeInBits() == 512)
12416 if (VT.getFixedSizeInBits() == 64)
12422 if (VT.isScalableVector()) {
12423 if (VT.getVectorElementType() != MVT::i1)
12427 if (VT == MVT::Other)
12429 uint64_t VTSize = VT.getFixedSizeInBits();
12445 if (VT.isScalableVector())
12447 if (VT.getSizeInBits() == 128)
12453 if (VT.isScalableVector())
12461 if (const auto *RegClass = getPredicateRegisterClass(*PC, VT))
12465 if (const auto *RegClass = getReducedGprRegisterClass(*RGC, VT))
12483 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
12496 if (VT != MVT::Other && VT.getSizeInBits() == 64) {
12674 EVT VT = V64Reg.getValueType();
12675 unsigned NarrowSize = VT.getVectorNumElements();
12676 MVT EltTy = VT.getVectorElementType().getSimpleVT();
12697 EVT VT = Op.getValueType();
12698 assert(!VT.isScalableVector() &&
12703 if (VT != MVT::v16i8 && VT != MVT::v8i8)
12706 unsigned NumElts = VT.getVectorNumElements();
12757 if (MaskSourceVec.getValueType() != VT)
12770 DAG.getUNDEF(VT));
12774 MaskSourceVec = DAG.getNode(ISD::AND, dl, VT, MaskSourceVec,
12775 DAG.getBuildVector(VT, dl, AndMaskConstants));
12778 ISD::INTRINSIC_WO_CHAIN, dl, VT,
12790 EVT VT = Op.getValueType();
12791 assert(!VT.isScalableVector() &&
12793 unsigned NumElts = VT.getVectorNumElements();
12853 unsigned OutputFactor = VT.getScalarSizeInBits() / 8;
12906 return DAG.getBitcast(VT, Shuffle);
12918 EVT SmallestEltTy = VT.getVectorElementType();
12926 VT.getScalarSizeInBits() / SmallestEltTy.getFixedSizeInBits();
12927 uint64_t VTSize = VT.getFixedSizeInBits();
13032 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
13045 VT.getScalarSizeInBits());
13072 V = DAG.getNode(AArch64ISD::NVCAST, dl, VT, Shuffle);
13074 V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
13085 static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
13086 unsigned NumElts = VT.getVectorNumElements();
13168 static bool isWideDUPMask(ArrayRef<int> M, EVT VT, unsigned BlockSize,
13173 if (BlockSize <= VT.getScalarSizeInBits())
13175 if (BlockSize % VT.getScalarSizeInBits() != 0)
13177 if (VT.getSizeInBits() % BlockSize != 0)
13180 size_t SingleVecNumElements = VT.getVectorNumElements();
13181 size_t NumEltsPerBlock = BlockSize / VT.getScalarSizeInBits();
13182 size_t NumBlocks = VT.getSizeInBits() / BlockSize;
13241 static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
13247 unsigned NumElts = VT.getVectorNumElements();
13284 static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
13285 unsigned NumElts = VT.getVectorNumElements();
13303 static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
13304 unsigned Half = VT.getVectorNumElements() / 2;
13322 static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
13323 unsigned NumElts = VT.getVectorNumElements();
13374 static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) {
13375 if (VT.getSizeInBits() != 128)
13378 unsigned NumElts = VT.getVectorNumElements();
13396 EVT VT = Op.getValueType();
13401 if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() ||
13402 VT.getVectorElementType() != V1.getValueType().getVectorElementType())
13407 if (!isConcatMask(Mask, VT, SplitV0))
13410 EVT CastVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
13419 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
13478 EVT VT = OpLHS.getValueType();
13492 if (VT.getScalarSizeInBits() == 16) {
13496 assert(VT.getScalarSizeInBits() == 32 &&
13507 if (VT == MVT::v4i16) {
13518 return DAG.getBitcast(VT, Ins);
13526 EVT VT = OpLHS.getValueType();
13533 if (VT.getVectorElementType() == MVT::i32 ||
13534 VT.getVectorElementType() == MVT::f32)
13535 return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS);
13537 if (VT.getVectorElementType() == MVT::i16 ||
13538 VT.getVectorElementType() == MVT::f16 ||
13539 VT.getVectorElementType() == MVT::bf16)
13540 return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS);
13542 assert(VT.getVectorElementType() == MVT::i8);
13543 return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS);
13548 EVT EltTy = VT.getVectorElementType();
13561 if (VT.getSizeInBits() == 64)
13564 return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
13570 return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
13574 return DAG.getNode(AArch64ISD::UZP1, dl, VT, OpLHS, OpRHS);
13576 return DAG.getNode(AArch64ISD::UZP2, dl, VT, OpLHS, OpRHS);
13578 return DAG.getNode(AArch64ISD::ZIP1, dl, VT, OpLHS, OpRHS);
13580 return DAG.getNode(AArch64ISD::ZIP2, dl, VT, OpLHS, OpRHS);
13582 return DAG.getNode(AArch64ISD::TRN1, dl, VT, OpLHS, OpRHS);
13584 return DAG.getNode(AArch64ISD::TRN2, dl, VT, OpLHS, OpRHS);
13675 static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
13724 unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2;
13725 Lane -= Idx * VT.getVectorNumElements() / 2;
13727 } else if (VT.getSizeInBits() == 64) {
13731 return DAG.getNode(Opcode, dl, VT, V, DAG.getConstant(Lane, dl, MVT::i64));
13746 EVT VT = Op.getValueType();
13747 EVT ScalarVT = VT.getVectorElementType();
13761 MVT NewEltVT = VT.isFloatingPoint()
13764 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
13768 return DAG.getBitcast(VT,
13786 EVT VT = Op.getValueType();
13812 SDValue TBLMask = DAG.getBuildVector(VT, dl, TBLMaskParts);
13828 EVT VT = Op.getValueType();
13831 assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&
13833 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
13838 return DAG.getBitcast(VT,
13845 EVT VT = Op.getValueType();
13849 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
13861 assert(V1.getValueType() == VT && "Unexpected VECTOR_SHUFFLE type!");
13862 assert(ShuffleMask.size() == VT.getVectorNumElements() &&
13881 return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane));
13885 return constructDup(V1, Lane, dl, VT, Opcode, DAG);
13891 if (isWideDUPMask(ShuffleMask, VT, LaneSize, Lane)) {
13897 unsigned NewEltCount = VT.getSizeInBits() / LaneSize;
13903 return DAG.getBitcast(VT, V1);
13907 unsigned NumElts = VT.getVectorNumElements();
13908 unsigned EltSize = VT.getScalarSizeInBits();
13918 SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
13919 return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev,
13925 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
13931 } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) {
13951 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
13955 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
13959 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
13982 EVT ScalarVT = VT.getVectorElementType();
13988 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
14018 assert(VT.getScalarSizeInBits() <= 32 &&
14024 EVT IVT = VT.changeVectorElementTypeToInteger();
14026 return DAG.getBitcast(VT, DAG.getNode(AArch64ISD::BSP, dl, IVT, MaskConst,
14037 EVT VT = Op.getValueType();
14039 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
14042 assert(VT.isScalableVector() && VT.getVectorElementType() == MVT::i1 &&
14058 if (VT == MVT::nxv1i1)
14063 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, ID, Zero, SplatVal);
14070 EVT VT = Op.getValueType();
14071 if (!isTypeLegal(VT) || !VT.isScalableVector())
14075 if (VT.getSizeInBits().getKnownMinValue() != AArch64::SVEBitsPerBlock)
14085 return DAG.getNode(AArch64ISD::DUPLANE128, DL, VT, Op.getOperand(1), CI);
14108 return DAG.getNode(ISD::BITCAST, DL, VT, TBL);
14114 EVT VT = BVN->getValueType(0);
14119 unsigned NumSplats = VT.getSizeInBits() / SplatBitSize;
14124 CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits());
14125 UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits());
14139 EVT VT = Op.getValueType();
14140 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64;
14148 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
14159 EVT VT = Op.getValueType();
14160 if (VT.isFixedLengthVector() &&
14166 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
14201 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
14212 EVT VT = Op.getValueType();
14213 if (VT.isFixedLengthVector() &&
14219 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
14246 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
14258 EVT VT = Op.getValueType();
14259 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
14277 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
14289 EVT VT = Op.getValueType();
14290 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
14298 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
14310 EVT VT = Op.getValueType();
14311 bool isWide = (VT.getSizeInBits() == 128);
14329 return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
14347 EVT VT = Bvec->getValueType(0);
14348 unsigned NumElts = VT.getVectorNumElements();
14410 EVT VT = N->getValueType(0);
14412 if (!VT.isVector())
14468 unsigned ElemSizeInBits = VT.getScalarSizeInBits();
14499 SDValue ResultSLI = DAG.getNode(Inst, DL, VT, X, Y, Imm);
14520 EVT VT = Op.getValueType();
14521 if (VT.isScalableVector())
14535 APInt DefBits(VT.getSizeInBits(), 0);
14536 APInt UndefBits(VT.getSizeInBits(), 0);
14563 EVT VT = Op.getValueType();
14564 EVT EltTy= VT.getVectorElementType();
14587 return DAG.getBuildVector(VT, dl, Ops);
14592 EVT VT = Op.getValueType();
14593 assert((VT.getSizeInBits() == 64 || VT.getSizeInBits() == 128) &&
14596 APInt DefBits(VT.getSizeInBits(), 0);
14597 APInt UndefBits(VT.getSizeInBits(), 0);
14632 assert(VT.getSizeInBits() % FVT.getScalarSizeInBits() == 0);
14634 .zext(VT.getSizeInBits());
14635 APInt NegBits(VT.getSizeInBits(), 0);
14636 unsigned NumElts = VT.getSizeInBits() / FVT.getScalarSizeInBits();
14647 AArch64ISD::NVCAST, DL, VT,
14665 EVT VT = Op.getValueType();
14667 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
14674 return convertFromScalableVector(DAG, VT, Seq);
14677 unsigned NumElems = VT.getVectorNumElements();
14678 if (!VT.isPow2VectorType() || VT.getFixedSizeInBits() > 128 ||
14688 if (VT.getScalarType().isInteger() &&
14719 return convertFromScalableVector(DAG, VT, Vec);
14724 EVT VT = Op.getValueType();
14728 if (useSVEForFixedLengthVectorVT(VT, OverrideNEON))
14745 unsigned BitSize = VT.getVectorElementType().getSizeInBits();
14748 if (Val.isZero() || (VT.isInteger() && Val.isAllOnes()))
14772 unsigned NumElts = VT.getVectorNumElements();
14836 return DAG.getUNDEF(VT);
14845 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
14869 if (VT.getVectorElementType() !=
14897 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
14900 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
14904 return DAG.getNode(AArch64ISD::UZP1, dl, VT, LHS, RHS);
14906 return DAG.getNode(AArch64ISD::UZP2, dl, VT, LHS, RHS);
14915 Value.getValueType() != VT) {
14918 return DAG.getNode(AArch64ISD::DUP, dl, VT, Value);
14932 unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
14933 return DAG.getNode(Opcode, dl, VT, Value, Lane);
14936 if (VT.getVectorElementType().isFloatingPoint()) {
14938 EVT EltTy = VT.getVectorElementType();
14953 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
14972 SDValue Val = DAG.getSplatBuildVector(VT, dl, ConstantValue);
14973 unsigned BitSize = VT.getScalarSizeInBits();
14982 Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue);
14990 // Note that type legalization likely mucked about with the VT of the
14992 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
15023 SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG);
15028 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector,
15066 EVT SubVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
15076 DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, DUP1, DUP2);
15102 SDValue VEC1 = DAG.getBuildVector(VT, dl, Ops1);
15103 SDValue VEC2 = DAG.getBuildVector(VT, dl, Ops2);
15105 DAG.getVectorShuffle(VT, dl, VEC1, VEC2, MaskVec);
15121 SDValue Vec = DAG.getUNDEF(VT);
15137 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0);
15149 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
15205 EVT VT = Op.getOperand(0).getValueType();
15207 if (VT.getScalarType() == MVT::i1) {
15208 EVT VectorVT = getPromotedVTForPredicate(VT);
15220 return DAG.getAnyExtOrTrunc(ExtendedVector, DL, VT);
15225 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
15235 EVT VT = Op.getOperand(0).getValueType();
15237 if (VT.getScalarType() == MVT::i1) {
15240 EVT VectorVT = getPromotedVTForPredicate(VT);
15250 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
15255 if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
15259 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
15260 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
15261 VT == MVT::v8f16 || VT == MVT::v8bf16)
15264 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
15265 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16 &&
15266 VT != MVT::v4bf16)
15286 EVT VT = Op.getValueType();
15287 assert(VT.isFixedLengthVector() &&
15296 assert(VT.is64BitVector() && "Extracting unexpected vector type!");
15321 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Container, Idx);
15331 return convertFromScalableVector(DAG, VT, Splice);
15348 EVT VT = Op.getValueType();
15351 if (!isTypeLegal(VT))
15355 if (VT.getVectorElementType() == MVT::i1) {
15356 unsigned NumElts = VT.getVectorMinNumElements();
15357 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
15371 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
15379 if (VT.getVectorElementCount() != (InVT.getVectorElementCount() * 2))
15385 EVT NarrowVT = getPackedSVEVectorVT(VT.getVectorElementCount());
15389 if (VT.isFloatingPoint()) {
15414 return getSVESafeBitCast(VT, Narrow, DAG);
15417 if (Idx == 0 && isPackedVectorType(VT, DAG)) {
15424 auto PredTy = VT.changeVectorElementType(MVT::i1);
15426 SDValue ScalableVec1 = convertToScalableVector(DAG, VT, Vec1);
15427 return DAG.getNode(ISD::VSELECT, DL, VT, PTrue, ScalableVec1, Vec0);
15465 EVT VT = Op.getValueType();
15468 if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
15471 assert(VT.isScalableVector() && "Expected a scalable vector.");
15479 SDValue Pg = getPredicateForScalableVector(DAG, dl, VT);
15481 DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, VT, Pg, Op->getOperand(0),
15484 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res);
15489 if (VT == MVT::nxv4i32 || VT == MVT::nxv2i64)
15495 if (VT == MVT::nxv16i8)
15497 else if (VT == MVT::nxv8i16)
15510 SDValue ResultLoCast = DAG.getNode(AArch64ISD::NVCAST, dl, VT, ResultLo);
15511 SDValue ResultHiCast = DAG.getNode(AArch64ISD::NVCAST, dl, VT, ResultHi);
15512 return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLoCast, ResultHiCast);
15516 EVT VT, unsigned DefinedValues) const {
15519 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
15522 bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
15524 if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
15527 if (VT.getVectorNumElements() == 4 &&
15528 (VT.is128BitVector() || VT.is64BitVector())) {
15538 unsigned EltSize = VT.getScalarSizeInBits();
15539 unsigned NumElts = VT.getVectorNumElements();
15540 return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
15544 isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
15548 isTRN_v_undef_Mask(M, VT, DummyUnsigned) ||
15549 isUZP_v_undef_Mask(M, VT, DummyUnsigned) ||
15550 isZIP_v_undef_Mask(M, VT, DummyUnsigned) ||
15552 isConcatMask(M, VT, VT.getSizeInBits() == 128));
15556 EVT VT) const {
15558 return isShuffleMaskLegal(M, VT);
15584 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
15585 assert(VT.isVector() && "vector shift count is not a vector type");
15586 int64_t ElementBits = VT.getScalarSizeInBits();
15595 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
15596 assert(VT.isVector() && "vector shift count is not a vector type");
15597 int64_t ElementBits = VT.getScalarSizeInBits();
15605 EVT VT = Op.getValueType();
15607 if (VT.getScalarType() == MVT::i1) {
15614 return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE);
15617 if (!VT.isVector() || VT.isScalableVector())
15637 EVT VT = Shift.getValueType();
15638 assert(VT.isScalableVT());
15653 assert(ResVT.getScalarSizeInBits() <= VT.getScalarSizeInBits() &&
15656 uint64_t ExtraBits = VT.getScalarSizeInBits() - ResVT.getScalarSizeInBits();
15674 EVT VT = Op.getValueType();
15680 unsigned EltSize = VT.getScalarSizeInBits();
15684 if (VT.isScalableVector() ||
15685 useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
15688 if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
15689 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
15691 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
15697 if (VT.isScalableVector() &&
15702 if (canLowerSRLToRoundingShiftForVT(Op, VT, DAG, ShiftValue, RShOperand))
15703 return DAG.getNode(AArch64ISD::URSHR_I_PRED, DL, VT,
15704 getPredicateForVector(DAG, DL, VT), RShOperand,
15708 if (VT.isScalableVector() ||
15709 useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable())) {
15716 if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
15719 return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
15729 SDValue NegShift = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
15732 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
15742 AArch64CC::CondCode CC, bool NoNans, EVT VT,
15745 assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
15769 Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
15771 Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
15772 return DAG.getNOT(dl, Fcmeq, VT);
15776 return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
15777 return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
15780 return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS);
15781 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
15784 return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS);
15785 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
15793 return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS);
15794 return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
15802 return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS);
15803 return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
15813 Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
15815 Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
15816 return DAG.getNOT(dl, Cmeq, VT);
15820 return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
15821 return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
15824 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS);
15825 return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS);
15828 return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS);
15830 return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS);
15831 return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS);
15834 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS);
15835 return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS);
15837 return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS);
15839 return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS);
15842 return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS);
15844 return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS);
15845 return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS);
15847 return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS);
15849 return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS);
15951 static SDValue getVectorBitwiseReduce(unsigned Opcode, SDValue Vec, EVT VT,
15986 return getVectorBitwiseReduce(Opcode, HalfVec, VT, DL, DAG);
16063 return DAG.getAnyExtOrTrunc(Result, DL, VT);
16150 MVT VT = Op.getSimpleValueType();
16151 assert(VT != MVT::i128 && "Handled elsewhere, code replicated.");
16154 RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getAllOnesConstant(dl, VT), RHS);
16171 EVT VT = Node->getValueType(0);
16179 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
16180 DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
16216 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
16217 DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
16237 EVT VT = Node->getValueType(0);
16244 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
16245 DAG.getConstant(-(uint64_t)Align->value(), dl, VT));
16277 EVT VT = Op.getValueType();
16278 assert(VT != MVT::i64 && "Expected illegal VSCALE node");
16283 VT);
16293 const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType());
16294 ElementCount EC = VT.getVectorElementCount();
16298 assert(VT == TLI.getMemValueType(DL, CI.getArgOperand(I)->getType()) &&
16301 // memVT is `NumVecs * VT`.
16302 Info.memVT = EVT::getVectorVT(CI.getType()->getContext(), VT.getScalarType(),
16526 EVT VT = Extend.getValueType();
16527 if ((VT == MVT::i64 || VT == MVT::i32) && Extend->use_size()) {
17616 auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {
17620 return allowsMisalignedMemoryAccesses(VT, 0, Align(1),
17646 auto AlignmentIsAcceptable = [&](EVT VT, Align AlignCheck) {
17650 return allowsMisalignedMemoryAccesses(VT, 0, Align(1),
17721 const EVT VT = AddNode.getValueType();
17722 if (VT.isVector() || VT.getScalarSizeInBits() > 64)
17735 unsigned BitSize = VT.getSizeInBits() <= 32 ? 32 : 64;
17848 const MachineFunction &MF, EVT VT) const {
17849 VT = VT.getScalarType();
17851 if (!VT.isSimple())
17854 switch (VT.getSimpleVT().SimpleTy) {
17879 EVT VT, CodeGenOptLevel OptLevel) const {
17880 return (OptLevel >= CodeGenOptLevel::Aggressive) && !VT.isScalableVector() &&
17881 !useSVEForFixedLengthVectorVT(VT);
17908 EVT VT = N->getValueType(0);
17920 if (ShiftLHS.getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) &&
17974 EVT VT = N->getValueType(0);
17975 if (N->getOpcode() == ISD::SRL && (VT == MVT::i32 || VT == MVT::i64)) {
18002 unsigned BinOpcode, EVT VT) const {
18003 return VT.isScalableVector() && isTypeLegal(VT);
18042 EVT VT = N->getValueType(0);
18043 if (!Subtarget->hasNEON() || !VT.isVector())
18060 return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0));
18290 EVT VT = A.getValueType();
18306 VT.getVectorNumElements() * 2)
18309 Ext1.getConstantOperandVal(1) != VT.getVectorNumElements()) &&
18311 Ext0.getConstantOperandVal(1) != VT.getVectorNumElements()))
18315 return DAG.getNode(Opcode, SDLoc(A), VT, Ext0.getOperand(0));
18338 EVT VT = A.getValueType();
18339 if (VT != MVT::v8i16 && VT != MVT::v4i32 && VT != MVT::v2i64)
18351 VT.getScalarSizeInBits() != (2 * ExtVT0.getScalarSizeInBits()))
18353 // Get VT for concat of zext sources.
18358 switch (VT.getSimpleVT().SimpleTy) {
18361 return DAG.getNode(AArch64ISD::UADDLV, SDLoc(A), VT, Concat);
18400 EVT VT = N->getValueType(0);
18404 if (VT.isScalableVector() ||
18405 (VT.isFixedLengthVector() && Subtarget->useSVEForFixedLengthVectors()))
18409 if ((VT != MVT::i32 && VT != MVT::i64) ||
18430 EVT VT = N->getValueType(0);
18434 if (VT.isScalableVector() || Subtarget->useSVEForFixedLengthVectors())
18438 if ((VT != MVT::i32 && VT != MVT::i64) ||
18448 SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
18449 SDValue Zero = DAG.getConstant(0, DL, VT);
18453 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne);
18454 CSNeg = DAG.getNode(AArch64ISD::CSNEG, DL, VT, And, And, CCVal, Cmp);
18460 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18463 SDValue AndPos = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne);
18464 SDValue AndNeg = DAG.getNode(ISD::AND, DL, VT, Negs, Pow2MinusOne);
18465 CSNeg = DAG.getNode(AArch64ISD::CSNEG, DL, VT, AndPos, AndNeg, CCVal,
18543 EVT VT = BV.getValueType();
18568 PreExtendType.getScalarSizeInBits() != VT.getScalarSizeInBits() / 2)
18597 EVT PreExtendVT = VT.changeVectorElementType(PreExtendType);
18607 EVT PreExtendVT = VT.changeVectorElementType(PreExtendType.getScalarType());
18617 return DAG.getNode(ExtOpc, DL, VT, NBV);
18624 EVT VT = Mul->getValueType(0);
18625 if (VT != MVT::v8i16 && VT != MVT::v4i32 && VT != MVT::v2i64)
18636 return DAG.getNode(Mul->getOpcode(), DL, VT, Op0 ? Op0 : Mul->getOperand(0),
18643 EVT VT = N->getValueType(0);
18644 if (VT != MVT::v2i64 && VT != MVT::v1i64 && VT != MVT::v2i32 &&
18645 VT != MVT::v4i32 && VT != MVT::v4i16 && VT != MVT::v8i16)
18660 unsigned HalfSize = VT.getScalarSizeInBits() / 2;
18667 VT.getVectorElementCount() * 2);
18672 return DAG.getNode(AArch64ISD::NVCAST, DL, VT, CM);
18680 EVT VT = N->getValueType(0);
18681 if (!VT.isFixedLengthVector() || VT.getSizeInBits() <= 128 ||
18699 EVT S2 = VT.getScalarType();
18705 VT.getVectorElementCount());
18711 DL, VT, NewOp);
18734 EVT VT = N->getValueType(0);
18754 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
18755 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal);
18759 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
18760 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
18815 return DAG.getNode(ISD::SHL, DL, VT, N0, RHS);
18820 return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
18825 return DAG.getNode(ISD::SUB, DL, VT, N0, N1);
18830 SDValue Zero = DAG.getConstant(0, DL, VT);
18831 return DAG.getNode(ISD::SUB, DL, VT, Zero, N);
18979 EVT VT = N->getValueType(0);
18980 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
18982 VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
19000 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
19005 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
19028 auto isSupportedType = [](EVT VT) {
19029 return !VT.isVector() && VT != MVT::bf16 && VT != MVT::f128;
19073 EVT VT = N->getValueType(0);
19074 if (VT != MVT::f32 && VT != MVT::f64)
19078 if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits())
19090 SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
19100 return DAG.getNode(Opcode, SDLoc(N), VT, Load);
19183 EVT VT = N->getValueType(0);
19188 if (!VT.isVector())
19191 if (VT.isScalableVector() && !Subtarget.hasSVE2())
19194 if (VT.isFixedLengthVector() &&
19195 (!Subtarget.isNeonAvailable() || TLI.useSVEForFixedLengthVectorVT(VT)))
19239 return DAG.getNode(AArch64ISD::BSP, DL, VT, Sub, SubSibling, AddSibling);
19246 unsigned Bits = VT.getScalarSizeInBits();
19255 return DAG.getNode(AArch64ISD::BSP, DL, VT, N0->getOperand(i),
19264 for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
19274 return DAG.getNode(AArch64ISD::BSP, DL, VT, N0->getOperand(i),
19292 EVT VT = N->getValueType(0);
19354 return DAG.getNode(AArch64ISD::CSEL, DL, VT, CSel0.getOperand(0),
19363 EVT VT = N->getValueType(0);
19368 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
19436 auto MaskAndTypeMatch = [ExtVal](EVT VT) -> bool {
19437 return ((ExtVal == 0xFF && VT == MVT::i8) ||
19438 (ExtVal == 0xFFFF && VT == MVT::i16) ||
19439 (ExtVal == 0xFFFFFFFF && VT == MVT::i32));
19535 EVT VT = N->getValueType(0);
19559 return DAG.getNode(AArch64ISD::CSINC, DL, VT, DAG.getConstant(0, DL, VT),
19560 DAG.getConstant(0, DL, VT),
19572 EVT VT = N->getValueType(0);
19580 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
19583 if (VT.isScalableVector())
19588 if (!VT.is64BitVector() && !VT.is128BitVector())
19599 APInt DefBits(VT.getSizeInBits(), 0);
19600 APInt UndefBits(VT.getSizeInBits(), 0);
19607 APInt ZeroSplat(VT.getSizeInBits(), 0);
19608 for (unsigned I = 0; I < VT.getSizeInBits() / Known.Zero.getBitWidth(); I++)
19609 ZeroSplat |= Known.Zero.zext(VT.getSizeInBits())
19635 EVT VT = N->getValueType(0);
19652 ISD::INTRINSIC_WO_CHAIN, DL, VT, A.getOperand(0),
19653 DAG.getNode(ISD::FADD, DL, VT, A.getOperand(1), B, N->getFlags()),
19666 static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16) {
19670 return (FullFP16 && VT == MVT::f16) || VT == MVT::f32 || VT == MVT::f64;
19672 return VT == MVT::i64;
19678 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
19711 EVT VT = N0.getValueType();
19713 if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1 ||
19724 SDValue Pg = getPTrue(DAG, SDLoc(N), VT, AArch64SVEPredPattern::all);
19777 EVT VT = N->getValueType(0);
19783 return VT.isInteger() ? DAG.getZExtOrTrunc(N0.getOperand(0), SDLoc(N), VT)
19795 if (isNullConstant(N1) && hasPairwiseAdd(N0->getOpcode(), VT, FullFP16) &&
19812 SDValue Extract1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
19814 SDValue Extract2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other,
19817 return DAG.getNode(N0->getOpcode(), DL, VT, Extract1, Extract2);
19824 {VT, MVT::Other},
19839 EVT VT = N->getValueType(0);
19843 if (VT.isScalableVector())
19866 N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) {
19871 return DAG.getNode(ISD::TRUNCATE, dl, VT,
19900 N000 = DAG.getNode(AArch64ISD::NVCAST, dl, VT, N000);
19901 N100 = DAG.getNode(AArch64ISD::NVCAST, dl, VT, N100);
19902 SDValue Uzp = DAG.getNode(AArch64ISD::UZP2, dl, VT, N000, N100);
19906 return DAG.getNode(AArch64ISD::VLSHR, dl, VT, Uzp, NewShiftConstant);
19970 DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
19975 VT);
19987 if (N->getNumOperands() == 2 && N0Opc == N1Opc && VT.is128BitVector() &&
19996 SDValue Concat0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N00, N10);
19997 SDValue Concat1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N01, N11);
19998 return DAG.getNode(N0Opc, dl, VT, Concat0, Concat1);
20006 EVT VT = Op.getValueType();
20008 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
20013 Imm = APInt(VT.getScalarSizeInBits(),
20018 Imm = APInt(VT.getScalarSizeInBits(),
20051 SDValue E0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(0),
20053 SDValue E1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(1),
20055 return DAG.getNode(AArch64ISD::ZIP1, dl, VT, E0, E1);
20061 if (N->getNumOperands() == 2 && N0 == N1 && VT.getVectorNumElements() == 2) {
20062 assert(VT.getScalarSizeInBits() == 64);
20063 return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG),
20089 return DAG.getNode(ISD::BITCAST, dl, VT,
20101 EVT VT = N->getValueType(0);
20102 if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1)
20113 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V.getOperand(0));
20224 MVT VT = N.getSimpleValueType();
20249 if (!VT.is64BitVector())
20253 unsigned NumElems = VT.getVectorNumElements();
20255 MVT ElementTy = VT.getVectorElementType();
20260 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N,
20405 EVT VT = Op->getValueType(0);
20406 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT));
20407 return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
20412 EVT VT = N->getValueType(0);
20414 if (N->getOpcode() != ISD::ADD || !VT.isScalarInteger())
20420 RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || LHS.getValueType() != VT)
20434 OpVT1.getVectorElementType() != VT)
20442 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
20451 EVT VT = N->getValueType(0);
20452 if (!VT.isScalarInteger() || N->getOpcode() != ISD::ADD)
20499 CTVal = cast<ConstantSDNode>(DAG.getConstant(C, DL, VT));
20500 CFVal = cast<ConstantSDNode>(DAG.getAllOnesConstant(DL, VT));
20515 SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, RHS, SDValue(CTVal, 0));
20519 return DAG.getNode(AArch64ISD::CSINC, DL, VT, NewNode, RHS, CCVal, Cmp);
20524 EVT VT = N->getValueType(0);
20541 return DAG.getNode(Dot.getOpcode(), SDLoc(N), VT, A, Dot.getOperand(1),
20551 EVT VT = Op.getValueType();
20552 SDValue Zero = DAG.getConstant(0, DL, VT);
20553 return DAG.getNode(ISD::SUB, DL, VT, Zero, Op);
20583 EVT VT = CSel.getValueType();
20584 return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0N, N1N, CSel.getOperand(2),
20605 MVT VT = N->getSimpleValueType(0);
20606 if (!VT.is128BitVector()) {
20629 RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS);
20635 LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS);
20638 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS);
20698 EVT VT = N->getValueType(0);
20703 return DAG.getNode(AArch64ISD::CSINC, DL, VT, LHS, LHS, CC, Cond);
20710 EVT VT = N->getValueType(0);
20713 (VT == MVT::v4f16 || VT == MVT::v4bf16)) {
20762 return DAG.getNode(ISD::FP_ROUND, DL, VT, Concat,
20769 if (VT == MVT::v2f64) {
20785 VT.getVectorMinNumElements() ==
20794 ISD::EXTRACT_SUBVECTOR, DL, VT.changeVectorElementType(MVT::f32),
20796 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Extract);
20803 // extract_vector_elt VT.
20810 if (VT != MVT::v2i32)
20814 // Reminder, EXTRACT_VECTOR_ELT has the effect of any-extending to its VT.
20826 Elt0->getConstantOperandVal(1) % VT.getVectorMinNumElements() == 0) {
20845 EVT VT = N->getValueType(0);
20847 if (VT.isFixedLengthVector() && VT.is64BitVector() && N0.hasOneUse() &&
20850 if (VT.getScalarType() == MVT::i32 &&
20853 return DAG.getNode(N0.getOpcode(), DL, VT, Op);
20867 assert((VT == MVT::i32 && N0.getValueType() == MVT::i64) &&
20881 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
20935 EVT VT = N->getValueType(0);
20938 SDValue NewSub = DAG.getNode(ISD::SUB, DL, VT, Z, Y);
20939 return DAG.getNode(ISD::ADD, DL, VT, NewSub, Shift);
20951 EVT VT = N->getValueType(0);
20952 if (VT != MVT::i32 && VT != MVT::i64)
20976 return DAG.getNode(ISD::ADD, DL, VT, RHS, LHS);
21006 EVT VT = N->getValueType(0);
21007 SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, X, M1);
21008 return DAG.getNode(ISD::SUB, SDLoc(N), VT, Sub, M2);
21069 EVT VT = N->getValueType(0);
21070 if (VT != MVT::i64 ||
21229 EVT VT = N->getValueType(0);
21230 if (!VT.isFixedLengthVector() ||
21231 (VT.getScalarSizeInBits() != 16 && VT.getScalarSizeInBits() != 32 &&
21232 VT.getScalarSizeInBits() != 64))
21264 if (NumSubElts * VT.getScalarSizeInBits() < 128 ||
21328 Ext0 = DAG.getNode(Other.getOpcode(), DL, VT, Extr0);
21329 Ext1 = DAG.getNode(Shift.getOperand(0).getOpcode(), DL, VT, Extr1);
21331 EVT DVT = VT.getDoubleNumVectorElementsVT(*DAG.getContext());
21333 SDValue SubL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Ext,
21336 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Ext,
21338 Ext0 = DAG.getVectorShuffle(VT, DL, SubL, SubH, LowMask);
21339 Ext1 = DAG.getVectorShuffle(VT, DL, SubL, SubH, HighMask);
21342 DAG.getNode(Shift.getOpcode(), DL, VT, Ext1, Shift.getOperand(1));
21343 return DAG.getNode(N->getOpcode(), DL, VT, Ext0, NShift);
21478 EVT VT = N->getValueType(0);
21481 if (VT == MVT::i64) {
21483 VT = MVT::v1i64;
21487 Op = DAG.getNode(Opcode, dl, VT, Op,
21494 Op = DAG.getNode(Opcode, dl, VT, Op,
21564 EVT VT = N->getValueType(0);
21566 assert(VT.isScalableVector() && "Expected a scalable vector.");
21569 if (VT.getSizeInBits().getKnownMinValue() != AArch64::SVEBitsPerBlock)
21572 unsigned ElemSize = VT.getVectorElementType().getSizeInBits() / 8;
21573 unsigned ByteSize = VT.getSizeInBits().getKnownMinValue() / 8;
21584 return DAG.getNode(ISD::BITCAST, dl, VT, EXT);
21597 EVT VT = N->getValueType(0);
21644 return DAG.getNode(AArch64ISD::SETCC_MERGE_ZERO, DL, VT, Pred,
21651 static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op,
21663 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
21686 return DAG.getZExtOrTrunc(Res, DL, VT);
22340 EVT VT = N->getValueType(0);
22341 if ((VT != MVT::v4i32 && VT != MVT::v8i16) ||
22347 if (ExtOffset != 0 && ExtOffset != VT.getVectorNumElements())
22353 InVT.getVectorNumElements() != VT.getVectorNumElements() * 2 ||
22354 InVT.getScalarSizeInBits() * 2 != VT.getScalarSizeInBits())
22359 Shuffle->getMask().slice(ExtOffset, VT.getVectorNumElements()), 4, Idx);
22368 Shuffle->getMask().slice(ExtOffset + VT.getVectorNumElements() / 2,
22369 VT.getVectorNumElements() / 2),
22374 SDValue BC1 = DAG.getNode(AArch64ISD::NVCAST, DL, VT,
22376 SDValue BC2 = DAG.getNode(AArch64ISD::NVCAST, DL, VT,
22379 VT, BC1, BC2);
22381 UZP = DAG.getNode(ISD::SRL, DL, VT, UZP,
22382 DAG.getConstant(InVT.getScalarSizeInBits(), DL, VT));
22384 ISD::AND, DL, VT, UZP,
22385 DAG.getConstant((1 << InVT.getScalarSizeInBits()) - 1, DL, VT));
22396 EVT VT = N->getValueType(0);
22398 (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16))
22409 APInt Mask = APInt::getLowBitsSet(VT.getScalarSizeInBits(),
22420 Mask = Mask.zext(VT.getScalarSizeInBits());
22424 Mask = Mask.zext(VT.getScalarSizeInBits());
22435 if (ExtOffset != 0 && ExtOffset != VT.getVectorNumElements())
22441 Shift += VT.getScalarSizeInBits() / 2;
22444 SDValue BC = DAG.getNode(AArch64ISD::NVCAST, DL, VT,
22447 BC = DAG.getNode(AArch64ISD::VLSHR, DL, VT, BC,
22449 return DAG.getNode(ISD::AND, DL, VT, BC, DAG.getConstant(Mask, DL, VT));
22576 EVT VT = N->getValueType(0);
22578 if (VT.getSizeInBits().getKnownMinValue() > AArch64::SVEBitsPerBlock)
22581 EVT ContainerVT = VT;
22589 DAG.getValueType(VT) };
22594 if (ContainerVT.isInteger() && (VT != ContainerVT))
22595 Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0));
22602 EVT VT = N->getValueType(0);
22605 EVT LoadVT = VT;
22606 if (VT.isFloatingPoint())
22607 LoadVT = VT.changeTypeToInteger();
22617 if (VT.isFloatingPoint()) {
22618 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) };
22631 EVT VT = N->getValueType(0);
22633 EVT LoadVT = VT;
22634 if (VT.isFloatingPoint())
22635 LoadVT = VT.changeTypeToInteger();
22641 if (VT.isFloatingPoint())
22642 Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0));
22707 EVT VT = StVal.getValueType();
22710 if (VT.isScalableVector())
22715 int NumVecElts = VT.getVectorNumElements();
22717 VT.getVectorElementType().getSizeInBits() == 64) ||
22719 VT.getVectorElementType().getSizeInBits() == 32)))
22755 if (VT.getVectorElementType().getSizeInBits() == 32) {
22774 EVT VT = StVal.getValueType();
22778 if (VT.isFloatingPoint())
22782 unsigned NumVecElts = VT.getVectorNumElements();
22834 EVT VT = StVal.getValueType();
22836 if (!VT.isFixedLengthVector())
22858 if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64)
22866 if (VT.getSizeInBits() != 128 || S->getAlign() >= Align(16) ||
22878 // Split VT into two.
22879 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
22931 EVT VT = N->getValueType(0);
22936 NumElts * VT.getVectorElementType().getSizeInBits() <= MinSVESize) {
22938 getPTrue(DAG, DL, VT.changeVectorElementType(MVT::i1), PgPattern);
22939 SDValue PassThru = DAG.getConstant(0, DL, VT);
22941 VT, DL, MLD->getChain(), MLD->getBasePtr(), MLD->getOffset(), Mask,
23011 EVT VT = Srl->getValueType(0);
23012 if (!VT.isScalableVector() || !Subtarget->hasSVE2())
23016 if (VT == MVT::nxv8i16)
23018 else if (VT == MVT::nxv4i32)
23020 else if (VT == MVT::nxv2i64)
23033 return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Rshrnb);
23329 auto VT = CC->getValueType(0).getHalfNumVectorElementsVT(*DAG.getContext());
23330 SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC,
23347 EVT VT = N->getValueType(0);
23349 if (!VT.is128BitVector() && !VT.is64BitVector())
23368 if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements())
23375 if (MemVT != VT.getVectorElementType())
23408 unsigned NumBytes = VT.getScalarSizeInBits() / 8;
23435 EVT Tys[3] = { VT, MVT::i64, MVT::Other };
23750 EVT VT = VecOp.getValueType();
23753 if (!MemVT.isVector() || !VT.isVector() ||
23837 auto hasValidElementTypeForFPTruncStore = [](EVT VT) {
23838 EVT EltVT = VT.getVectorElementType();
24705 EVT VT = Op->getValueType(0);
24708 return DAG.getNode(AArch64ISD::CSEL, DL, VT, L, R, CCValue, Cond);
24723 EVT VT = N->getValueType(0);
24769 DAG.getVTList(VT, MVT_CC), CmpOpOther, SubsOp);
24774 SDValue Res = DAG.getNode(ISD::ADD, SDLoc(N->getOperand(OpNum)), VT,
24782 return DAG.getNode(AArch64ISD::CSEL, SDLoc(N), VT, TValReassoc, FValReassoc,
24973 EVT VT = N->getValueType(0);
24993 return DAG.getZExtOrTrunc(CSEL, DL, VT);
24999 LHS->getConstantOperandVal(1) < VT.getScalarSizeInBits() &&
25007 return DAG.getNode(ISD::SETCC, DL, VT, TST, RHS, N->getOperand(2));
25015 if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
25028 return DAG.getSetCC(DL, VT, LHS, RHS, Cond);
25047 EVT VT = N->getValueType(0);
25051 SDValue Res = DCI.DAG.getNode(GenericOpcode, DL, VT, N->ops());
25058 GenericOpcode, DCI.DAG.getVTList(VT), {LHS, RHS}))
25311 EVT VT = CmpLHS.getValueType();
25317 VT.isSimple() &&
25320 VT.getSimpleVT().SimpleTy) &&
25324 unsigned NumElts = VT.getVectorNumElements();
25326 NumElts, DAG.getConstant(VT.getScalarSizeInBits() - 1, SDLoc(N),
25327 VT.getScalarType()));
25328 SDValue Val = DAG.getBuildVector(VT, SDLoc(N), Ops);
25330 auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val);
25331 auto Or = DAG.getNode(ISD::OR, SDLoc(N), VT, Shift, N->getOperand(1));
25429 EVT VT = N->getValueType(0);
25433 if (VT.is64BitVector() && DCI.isAfterLegalizeDAG()) {
25434 EVT LVT = VT.getDoubleNumVectorElementsVT(*DCI.DAG.getContext());
25438 return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0),
25454 if (VT == EXTRACT_VEC_ELT.getOperand(0).getValueType()) {
25455 unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
25456 return DCI.DAG.getNode(Opcode, DL, VT, EXTRACT_VEC_ELT.getOperand(0),
25832 auto VT = cast<VTSDNode>(N->getOperand(1))->getVT();
25833 EVT EltTy = VT.getVectorElementType();
25839 EVT ExtVT = VT.getDoubleNumVectorElementsVT(*DAG.getContext());
26058 EVT VT = N->getValueType(0);
26064 auto hasValidElementTypeForFPExtLoad = [](EVT VT) {
26065 EVT EltVT = VT.getVectorElementType();
26074 VT.isFixedLengthVector() && hasValidElementTypeForFPExtLoad(VT) &&
26075 VT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits()) {
26077 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
26094 EVT VT = N->getValueType(0);
26097 if (!VT.isScalableVector() || Subtarget->hasSVE2() || Subtarget->hasSME())
26106 SDValue InvMask = DAG.getNOT(DL, Mask, VT);
26107 SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1);
26108 SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2);
26109 return DAG.getNode(ISD::OR, DL, VT, Sel, SelInv);
26113 EVT VT = N->getValueType(0);
26144 return DAG.getNode(ISD::BITCAST, DL, VT, NewDuplane128);
26311 EVT VT = N->getValueType(0);
26312 if (VT != MVT::v1i64)
26374 EVT VT = N->getValueType(0);
26379 SDValue NewRHS = DAG.getNode(ISD::SHL, DL, VT, C1, C2);
26384 SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, X, C2);
26385 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewRHS);
26850 EVT VT;
26853 VT = LD->getMemoryVT();
26856 VT = ST->getMemoryVT();
26870 EVT VT;
26873 VT = LD->getMemoryVT();
26876 VT = ST->getMemoryVT();
26896 EVT VT = N->getValueType(0);
26915 Results.push_back(DAG.getZExtOrTrunc(VectorBits, DL, VT));
26924 EVT VT = N->getValueType(0);
26931 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, CastVal, IdxZero));
26938 EVT VT = N->getValueType(0);
26941 if (VT == MVT::v2i16 && SrcVT == MVT::i32) {
26946 if (VT == MVT::v4i8 && SrcVT == MVT::i32) {
26951 if (VT == MVT::v2i8 && SrcVT == MVT::i16) {
26956 if (VT.isScalableVector() && !isTypeLegal(VT) && isTypeLegal(SrcVT)) {
26957 assert(!VT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
26965 if (VT.getVectorElementCount() != SrcVT.getVectorElementCount())
26968 SDValue CastResult = getSVESafeBitCast(getSVEContainerType(VT), Op, DAG);
26969 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, CastResult));
26974 !VT.isVector())
26977 if (VT != MVT::i16 || (SrcVT != MVT::f16 && SrcVT != MVT::bf16))
26989 EVT VT = N->getValueType(0);
26990 if (!VT.is256BitVector() ||
26991 (VT.getScalarType().isFloatingPoint() &&
26993 (VT.getScalarType() == MVT::f16 && !Subtarget->hasFullFP16()) ||
26994 VT.getScalarType() == MVT::bf16)
27023 for (unsigned I = 0, E = VT.getVectorNumElements() / 2; I < E; I++) {
27028 DAG.getVectorShuffle(VT, DL,
27029 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Addp,
27031 DAG.getUNDEF(VT), NMask));
27058 EVT VT = N->getValueType(0);
27062 ElementCount ResEC = VT.getVectorElementCount();
27076 EVT ExtendedHalfVT = VT.widenIntegerVectorElementType(*DAG.getContext());
27079 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half));
27464 EVT VT = N->getValueType(0);
27472 assert((VT == MVT::i8 || VT == MVT::i16) &&
27478 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
27482 assert((VT == MVT::i8 || VT == MVT::i16) &&
27488 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
27492 assert((VT == MVT::i8 || VT == MVT::i16) &&
27497 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
27501 assert((VT == MVT::i8 || VT == MVT::i16) &&
27506 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
27520 if (!VT.isFixedLengthVector() || VT.getVectorElementType() != MVT::i1)
27524 EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
27525 if (NewVT.getVectorNumElements() != VT.getVectorNumElements())
27530 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V));
27570 AArch64TargetLowering::getPreferredVectorAction(MVT VT) const {
27573 if (VT == MVT::v1i8 || VT == MVT::v1i16 || VT == MVT::v1i32 ||
27574 VT == MVT::v1f32)
27577 return TargetLoweringBase::getPreferredVectorAction(VT);
28096 bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
28105 return OptSize && !VT.isVector();
28123 bool AArch64TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
28125 return VT.isScalarInteger();
28129 EVT VT) const {
28136 return TargetLowering::shouldConvertFpToSat(Op, FPVT, VT);
28139 bool AArch64TargetLowering::shouldExpandCmpUsingSelects(EVT VT) const {
28142 return !VT.isFixedLengthVector();
28174 bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const {
28175 return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint();
28325 // Return the largest legal scalable vector type that matches VT's element type.
28326 static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT) {
28327 assert(VT.isFixedLengthVector() &&
28328 DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
28330 switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
28352 // Return a PTRUE with active lanes corresponding to the extent of VT.
28354 EVT VT) {
28355 assert(VT.isFixedLengthVector() &&
28356 DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
28360 getSVEPredPatternFromNumElements(VT.getVectorNumElements());
28370 MaxSVESize == VT.getSizeInBits())
28374 switch (VT.getVectorElementType().getSimpleVT().SimpleTy) {
28399 EVT VT) {
28400 assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
28402 auto PredTy = VT.changeVectorElementType(MVT::i1);
28406 static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT) {
28407 if (VT.isFixedLengthVector())
28408 return getPredicateForFixedLengthVector(DAG, DL, VT);
28410 return getPredicateForScalableVector(DAG, DL, VT);
28414 static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
28415 assert(VT.isScalableVector() &&
28421 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
28424 // Shrink V so it's just big enough to maintain a VT's worth of data.
28425 static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V) {
28426 assert(VT.isFixedLengthVector() &&
28432 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
28441 EVT VT = Op.getValueType();
28442 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28446 auto Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
28448 if (VT.isFloatingPoint()) {
28459 if (VT.isFloatingPoint() && Load->getExtensionType() == ISD::EXTLOAD) {
28466 } else if (VT.isFloatingPoint()) {
28470 Result = convertFromScalableVector(DAG, VT, Result);
28499 EVT VT = Op.getValueType();
28500 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28505 if (VT.getScalarSizeInBits() > Mask.getValueType().getScalarSizeInBits()) {
28508 Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Mask);
28539 Result = convertFromScalableVector(DAG, VT, Result);
28550 EVT VT = Store->getValue().getValueType();
28551 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28554 auto Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
28557 if (VT.isFloatingPoint() && Store->isTruncatingStore()) {
28566 } else if (VT.isFloatingPoint()) {
28583 EVT VT = Store->getValue().getValueType();
28584 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28598 EVT VT = Op.getValueType();
28599 EVT EltVT = VT.getVectorElementType();
28607 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28611 SDValue Pg = getPredicateForFixedLengthVector(DAG, dl, VT);
28618 return convertFromScalableVector(DAG, VT, Res);
28626 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
28631 EVT WideVT = VT.widenIntegerVectorElementType(*DAG.getContext());
28636 return DAG.getNode(ISD::TRUNCATE, dl, VT, Div);
28658 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, {LoTrunc, HiTrunc});
28663 EVT VT = Op.getValueType();
28664 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
28680 if (VT.getVectorElementType() == MVT::i16)
28685 if (VT.getVectorElementType() == MVT::i32)
28690 assert(VT.getVectorElementType() == MVT::i64 && "Unexpected element type!");
28694 return convertFromScalableVector(DAG, VT, Val);
28699 EVT VT = Op.getValueType();
28700 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
28714 if (VT.getVectorElementType() == MVT::i32)
28720 if (VT.getVectorElementType() == MVT::i16)
28726 assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
28730 return convertFromScalableVector(DAG, VT, Val);
28735 EVT VT = Op.getValueType();
28743 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1));
28748 EVT VT = Op.getValueType();
28749 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
28759 return convertFromScalableVector(DAG, VT, ScalableRes);
28768 EVT VT = Op.getValueType();
28770 auto Pg = getPredicateForVector(DAG, DL, VT);
28772 if (VT.isFixedLengthVector()) {
28773 assert(isTypeLegal(VT) && "Expected only legal fixed-width types");
28774 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28800 return convertFromScalableVector(DAG, VT, ScalableRes);
28803 assert(VT.isScalableVector() && "Only expect to lower scalable vector op!");
28814 Operands.push_back(DAG.getUNDEF(VT));
28816 return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags());
28824 EVT VT = Op.getValueType();
28825 assert(VT.isFixedLengthVector() && isTypeLegal(VT) &&
28827 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
28848 return convertFromScalableVector(DAG, VT, ScalableRes);
28884 EVT VT = ReduceOp.getValueType();
28898 return getPTest(DAG, VT, Op, Op, AArch64CC::ANY_ACTIVE);
28900 return getPTest(DAG, VT, Pg, Op, AArch64CC::ANY_ACTIVE);
28903 return getPTest(DAG, VT, Pg, Op, AArch64CC::NONE_ACTIVE);
28915 return DAG.getAnyExtOrTrunc(Cntp, DL, VT);
28972 EVT VT = Op.getValueType();
28991 return convertFromScalableVector(DAG, VT, ScalableRes);
29023 EVT VT = Op.getValueType();
29024 EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT);
29030 return convertFromScalableVector(DAG, VT, Op);
29043 EVT VT = Op.getValueType();
29053 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops);
29056 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
29064 return convertFromScalableVector(DAG, VT, Op);
29070 EVT VT = Op.getValueType();
29071 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
29075 SDValue Pg = getPredicateForVector(DAG, DL, VT);
29077 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
29082 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT.changeTypeToInteger(), Val);
29089 return convertFromScalableVector(DAG, VT, Val);
29095 EVT VT = Op.getValueType();
29096 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
29103 VT.getVectorElementType());
29112 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val);
29113 return DAG.getNode(ISD::BITCAST, DL, VT, Val);
29119 EVT VT = Op.getValueType();
29120 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
29129 EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT);
29132 if (VT.bitsGE(SrcVT)) {
29133 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
29136 VT.changeTypeToInteger(), Val);
29144 return convertFromScalableVector(DAG, VT, Val);
29155 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val);
29156 return DAG.getNode(ISD::BITCAST, DL, VT, Val);
29256 EVT VT = Op.getValueType();
29257 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
29266 EVT ContainerDstVT = getContainerForFixedLengthVector(DAG, VT);
29269 if (VT.bitsGT(SrcVT)) {
29272 SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
29275 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Val);
29281 return convertFromScalableVector(DAG, VT, Val);
29292 return DAG.getNode(ISD::TRUNCATE, DL, VT, Val);
29297 ArrayRef<int> ShuffleMask, EVT VT,
29404 Shuffle = convertFromScalableVector(DAG, VT, Shuffle);
29410 EVT VT = Op.getValueType();
29411 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
29420 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
29432 EVT ScalarTy = MinLegalExtractEltScalarTy(VT.getVectorElementType());
29436 return convertFromScalableVector(DAG, VT, Op);
29441 if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm) &&
29442 Imm == VT.getVectorNumElements() - 1) {
29445 EVT ScalarTy = MinLegalExtractEltScalarTy(VT.getVectorElementType());
29448 DAG.getConstant(VT.getVectorNumElements() - 1, DL, MVT::i64));
29450 return convertFromScalableVector(DAG, VT, Op);
29453 unsigned EltSize = VT.getScalarSizeInBits();
29455 if (isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), LaneSize)) {
29469 return convertFromScalableVector(DAG, VT, Op);
29474 isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), 128)) {
29475 if (!VT.isFloatingPoint())
29482 return convertFromScalableVector(DAG, VT, Op);
29486 if (isZIPMask(ShuffleMask, VT.getVectorNumElements(), WhichResult) &&
29489 DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op2));
29491 if (isTRNMask(ShuffleMask, VT.getVectorNumElements(), WhichResult)) {
29494 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2));
29497 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult == 0)
29499 DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1));
29501 if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
29504 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
29527 if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) {
29531 return convertFromScalableVector(DAG, VT, Op);
29534 if (isZIPMask(ShuffleMask, VT.getVectorNumElements(), WhichResult) &&
29537 DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op2));
29539 if (isUZPMask(ShuffleMask, VT.getVectorNumElements(), WhichResult)) {
29542 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2));
29545 if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult != 0)
29547 DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1));
29549 if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
29552 DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
29565 return GenerateFixedLengthSVETBL(Op, Op1, Op2, ShuffleMask, VT, ContainerVT,
29571 SDValue AArch64TargetLowering::getSVESafeBitCast(EVT VT, SDValue Op,
29576 assert(VT.isScalableVector() && isTypeLegal(VT) &&
29579 assert(VT.getVectorElementType() != MVT::i1 &&
29583 if (InVT == VT)
29586 EVT PackedVT = getPackedSVEVectorVT(VT.getVectorElementType());
29596 assert((VT.getVectorElementCount() == InVT.getVectorElementCount() ||
29597 VT == PackedVT || InVT == PackedInVT) &&
29622 if (VT != PackedVT)
29623 Op = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
29633 EVT AArch64TargetLowering::getPromotedVTForPredicate(EVT VT) const {
29634 return ::getPromotedVTForPredicate(VT);
29895 EVT VT) const {
29897 VT.isFixedLengthVector() && !VT.getVectorElementCount().isScalar();
29899 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
29904 getVectorTypeBreakdownForCallingConv(Context, CC, VT, VT1, NumIntermediates,
29910 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
29912 VT.isFixedLengthVector() && !VT.getVectorElementCount().isScalar();
29914 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
29919 return getVectorTypeBreakdownForCallingConv(Context, CC, VT, VT1,
29924 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
29927 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
29933 assert(IntermediateVT == RegisterVT && "Unexpected VT mismatch!");
29938 if (RegisterVT.getSizeInBits() * NumRegs != VT.getSizeInBits()) {
29939 EVT EltTy = VT.getVectorElementType();
29945 NumIntermediates = VT.getVectorNumElements();
29995 bool AArch64TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
30000 if (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32)
30004 return TargetLowering::isTypeDesirableForOp(Opc, VT);
30018 EVT VT = N->getValueType(0);
30021 assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&
30022 VT.isInteger() && Op0VT.isInteger() && Op1VT.isInteger() &&
30024 assert(VT == Op0VT &&
30039 EVT VT = N->getValueType(0);
30041 assert(OpVT.isVector() && VT.isVector() && OpVT.isInteger() &&
30042 VT.isInteger() && "Expected integer vectors!");
30043 assert(OpVT.getSizeInBits() == VT.getSizeInBits() &&
30045 assert(OpVT.getVectorElementCount() == VT.getVectorElementCount() * 2 &&
30057 EVT VT = N->getValueType(0);
30060 assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&
30062 assert(VT == Op0VT && VT == Op1VT && "Expected matching vectors!");
30068 EVT VT = N->getValueType(0);
30071 assert(VT.isVector() && VT.isInteger() &&
30075 assert(VT.getSizeInBits() == Op0VT.getSizeInBits() &&
30077 assert(VT.getVectorElementCount() == Op0VT.getVectorElementCount() * 2 &&