Lines Matching +full:sense +full:- +full:bitfield +full:- +full:width
1 //===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
31 #define DEBUG_TYPE "aarch64-isel"
34 //===--------------------------------------------------------------------===//
35 /// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
42 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
60 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
137 int64_t C = CI->getSExtValue();
145 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
149 template<int Width>
152 return SelectAddrModeWRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
155 template<int Width>
158 return SelectAddrModeXRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
162 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
163 N = N->getOperand(0);
164 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
165 !isa<ConstantSDNode>(N->getOperand(1)))
167 EVT VT = N->getValueType(0);
168 EVT LVT = N->getOperand(0).getValueType();
169 unsigned Index = N->getConstantOperandVal(1);
173 Res = N->getOperand(0);
180 SDValue Op = N->getOperand(0);
182 unsigned ShtAmt = N->getConstantOperandVal(1);
198 if (Imm != 1ULL << (ShtAmt - 1))
202 Res2 = CurDAG->getTargetConstant(ShtAmt, SDLoc(N), MVT::i32);
207 switch(N->getOpcode()) {
212 auto Opnd0 = N->getOperand(0);
227 switch(N->getOpcode()) {
230 auto Opnd0 = N->getOperand(0);
243 switch(N->getOpcode()) {
246 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
247 return Const && Const->isZero() && Const->isNegative();
285 if (N->getOpcode() != ISD::SPLAT_VECTOR)
288 EVT EltVT = N->getValueType(0).getVectorElementType();
289 return SelectSVEShiftImm(N->getOperand(0), /* Low */ 1,
300 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
309 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32);
321 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
325 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32);
335 uint64_t C = CI->getZExtValue();
340 Imm = CurDAG->getRegister(BaseReg + C, MVT::Other);
346 /// Form sequences of consecutive 64/128-bit registers for use in NEON
347 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
357 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
482 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
490 unsigned Width);
528 /// isIntImmediate - This method tests to see if the node is a constant
529 /// operand. If so Imm will receive the 32-bit value.
532 Imm = C->getZExtValue();
538 // isIntImmediate - This method tests to see if a constant operand.
544 // isOpcWithIntImmediate - This method tests to see if the node is a specific
549 return N->getOpcode() == Opc &&
550 isIntImmediate(N->getOperand(1).getNode(), Imm);
553 // isIntImmediateEq - This method tests to see if N is a constant operand that
575 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
576 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF);
578 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64);
580 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
589 /// SelectArithImmed - Select an immediate value that can be represented as
590 /// a 12-bit value shifted left by either 0 or 12. If so, return true with
591 /// Val set to the 12-bit value and Shift set to the shifter operand.
598 // root-level opcode matching.
602 uint64_t Immed = N.getNode()->getAsZExtVal();
615 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
616 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
620 /// SelectNegArithImmed - As above, but negates the value before trying to
628 // root-level opcode matching.
632 // The immediate operand must be a 24-bit zero-extended immediate.
633 uint64_t Immed = N.getNode()->getAsZExtVal();
649 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val,
653 /// getShiftTypeForNode - Translate a shift node to the corresponding
678 unsigned ShiftVal = CSD->getZExtValue();
682 // Check if this particular node is reused in any non-memory related
686 for (SDNode *UI : Node->uses())
688 for (SDNode *UII : UI->uses())
699 if (CurDAG->shouldOptForSize() || V.hasOneUse())
703 // costs additional micro-ops.
704 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
708 // it's used by a non-address operation.
724 /// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
732 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
735 if (!LHS->hasOneUse())
738 unsigned LHSOpcode = LHS->getOpcode();
746 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
751 APInt AndMask = RHSC->getAPIntValue();
766 NewShiftC = LowZBits - ShiftAmtC;
792 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC, DL, VT);
793 SDValue BitWidthMinus1 = CurDAG->getTargetConstant(BitWidth - 1, DL, VT);
794 Reg = SDValue(CurDAG->getMachineNode(NewShiftOp, DL, VT, LHS->getOperand(0),
798 Shift = CurDAG->getTargetConstant(ShVal, DL, MVT::i32);
802 /// getExtendTypeForNode - Translate an extend node to the corresponding
810 SrcVT = cast<VTSDNode>(N.getOperand(1))->getVT();
820 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
832 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
839 uint64_t AndMask = CSD->getZExtValue();
863 if (CurDAG->shouldOptForSize() || V.hasOneUse())
868 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
877 /// SelectShiftedRegister - Select a "shifted register" operand. If the value
895 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
899 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32);
915 return CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl, MVT::i32, N);
924 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
928 Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32);
936 /// SelectArithExtendedRegister - Select a "extended register" operand. This
947 ShiftVal = CSD->getZExtValue();
963 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
972 if (Ext == AArch64_AM::UXTW && Reg->getValueType(0).getSizeInBits() == 32 &&
980 // there might not be an actual 32-bit value in the program. We can
984 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
989 /// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1002 ShiftVal = CSD->getZExtValue();
1008 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
1016 /// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1019 for (auto *Use : N->uses()) {
1020 if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
1021 Use->getOpcode() != ISD::ATOMIC_LOAD &&
1022 Use->getOpcode() != ISD::ATOMIC_STORE)
1027 if (isStrongerThanMonotonic(cast<MemSDNode>(Use)->getSuccessOrdering()))
1037 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1043 /// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1051 const DataLayout &DL = CurDAG->getDataLayout();
1054 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1055 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1056 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1060 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1062 if (CurDAG->isBaseWithConstantOffset(N)) {
1065 int64_t RHSC = RHS->getSExtValue();
1067 int64_t Range = 0x1LL << (BW - 1);
1069 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1073 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1074 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1076 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1081 uint64_t RHSC = RHS->getZExtValue();
1085 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1088 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1089 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1091 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1102 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1106 /// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1112 const DataLayout &DL = CurDAG->getDataLayout();
1115 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1116 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1117 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1129 if (GAN->getOffset() % Size == 0 &&
1130 GAN->getGlobal()->getPointerAlignment(DL) >= Size)
1134 if (CurDAG->isBaseWithConstantOffset(N)) {
1136 int64_t RHSC = (int64_t)RHS->getZExtValue();
1141 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1142 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1144 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1160 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1164 /// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1172 if (!CurDAG->isBaseWithConstantOffset(N))
1175 int64_t RHSC = RHS->getSExtValue();
1176 if (RHSC >= -256 && RHSC < 256) {
1179 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1181 Base = CurDAG->getTargetFrameIndex(
1182 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1184 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64);
1194 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1195 return CurDAG->getTargetInsertSubreg(AArch64::sub_32, dl, MVT::i64, ImpDef,
1206 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1217 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1221 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1225 unsigned ShiftVal = CSD->getZExtValue();
1244 // to the register-immediate addressing modes.
1248 // Check if this particular node is reused in any non-memory related
1252 for (SDNode *UI : Node->uses()) {
1264 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
1272 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
1277 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32);
1286 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1298 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1332 // Check if this particular node is reused in any non-memory related
1336 for (SDNode *UI : Node->uses()) {
1353 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1356 // checked by using -ImmOff).
1358 isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
1363 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
1366 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV);
1376 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1384 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1388 // Match any non-shifted, non-extend, non-immediate add expression.
1391 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32);
1392 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32);
1440 // There's no special register-class for a vector-list of 1 element: it's just
1453 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32));
1455 // Then we get pairs of source & subregister-position for the components.
1458 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32));
1462 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
1469 EVT VT = N->getValueType(0);
1475 SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Off,
1476 N->op_begin() + Vec0Off + NumVecs);
1481 Ops.push_back(N->getOperand(1));
1483 Ops.push_back(N->getOperand(NumVecs + ExtOff + 1));
1484 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1494 // Otherwise, it's either a constant discriminator, or a non-blended
1496 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1497 Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) {
1498 AddrDisc = Disc->getOperand(1);
1499 ConstDisc = Disc->getOperand(2);
1505 // discriminator value) isn't a 16-bit constant, bail out, and let the
1508 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1509 return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc);
1513 AddrDisc = DAG->getRegister(AArch64::XZR, MVT::i64);
1516 DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64),
1523 SDValue Val = N->getOperand(1);
1524 SDValue AUTKey = N->getOperand(2);
1525 SDValue AUTDisc = N->getOperand(3);
1527 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1528 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
1534 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
1538 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT, DL, MVT::i64, Ops);
1546 SDValue Val = N->getOperand(1);
1547 SDValue AUTKey = N->getOperand(2);
1548 SDValue AUTDisc = N->getOperand(3);
1549 SDValue PACKey = N->getOperand(4);
1550 SDValue PACDisc = N->getOperand(5);
1552 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1553 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
1555 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
1556 PACKey = CurDAG->getTargetConstant(PACKeyC, DL, MVT::i64);
1566 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
1572 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC, DL, MVT::i64, Ops);
1579 if (LD->isUnindexed())
1581 EVT VT = LD->getMemoryVT();
1582 EVT DstVT = N->getValueType(0);
1583 ISD::MemIndexedMode AM = LD->getAddressingMode();
1591 ISD::LoadExtType ExtType = LD->getExtensionType();
1645 SDValue Chain = LD->getChain();
1646 SDValue Base = LD->getBasePtr();
1647 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
1648 int OffsetVal = (int)OffsetOp->getZExtValue();
1650 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1652 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1656 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
1657 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {MemOp});
1662 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1664 SDValue(CurDAG->getMachineNode(
1666 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1674 CurDAG->RemoveDeadNode(N);
1681 EVT VT = N->getValueType(0);
1682 SDValue Chain = N->getOperand(0);
1684 SDValue Ops[] = {N->getOperand(2), // Mem operand;
1689 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1693 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1700 MachineMemOperand *MemOp = MemIntr->getMemOperand();
1701 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
1704 CurDAG->RemoveDeadNode(N);
1710 EVT VT = N->getValueType(0);
1711 SDValue Chain = N->getOperand(0);
1713 SDValue Ops[] = {N->getOperand(1), // Mem operand
1714 N->getOperand(2), // Incremental
1720 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1732 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1736 CurDAG->RemoveDeadNode(N);
1751 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
1771 /// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
1804 case 16: // 8-bit or bf16
1807 case 8: // 16-bit
1810 case 4: // 32-bit
1813 case 2: // 64-bit
1828 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(N->getOperand(2)))
1829 if (Imm->getZExtValue() > 1)
1833 EVT VT = N->getValueType(0);
1834 SDValue Ops[] = {N->getOperand(1), N->getOperand(2)};
1835 SDNode *WhilePair = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops);
1839 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg(
1842 CurDAG->RemoveDeadNode(N);
1847 EVT VT = N->getValueType(0);
1849 SDValue Ops[] = {N->getOperand(1), N->getOperand(2)};
1851 SDNode *WhilePair = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops);
1855 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg(
1858 CurDAG->RemoveDeadNode(N);
1863 EVT VT = N->getValueType(0);
1864 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1867 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, Ops);
1870 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
1873 CurDAG->RemoveDeadNode(N);
1884 EVT VT = N->getValueType(0);
1888 SmallVector<SDValue, 4> Regs(N->op_begin() + StartIdx,
1889 N->op_begin() + StartIdx + NumVecs);
1899 Zm = N->getOperand(NumVecs + FirstVecIdx);
1903 Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped,
1904 N->getOperand(1), Zdn, Zm);
1906 Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, Zdn, Zm);
1909 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
1912 CurDAG->RemoveDeadNode(N);
1920 EVT VT = N->getValueType(0);
1921 SDValue Chain = N->getOperand(0);
1927 N, Opc_rr, Opc_ri, N->getOperand(IsIntr ? 3 : 2),
1928 CurDAG->getTargetConstant(0, DL, MVT::i64), Scale);
1930 SDValue Ops[] = {N->getOperand(IsIntr ? 2 : 1), // Predicate
1936 SDNode *Load = CurDAG->getMachineNode(Opc, DL, ResTys, Ops);
1939 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
1945 CurDAG->RemoveDeadNode(N);
1955 EVT VT = N->getValueType(0);
1956 SDValue Chain = N->getOperand(0);
1958 SDValue PNg = N->getOperand(2);
1959 SDValue Base = N->getOperand(3);
1960 SDValue Offset = CurDAG->getTargetConstant(0, DL, MVT::i64);
1965 SDValue Ops[] = {PNg, // Predicate-as-counter
1971 SDNode *Load = CurDAG->getMachineNode(Opc, DL, ResTys, Ops);
1974 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
1980 CurDAG->RemoveDeadNode(N);
1985 if (N->getValueType(0) != MVT::nxv4f32)
1993 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Node->getOperand(4)))
1994 if (Imm->getZExtValue() > MaxImm)
1998 if (!ImmToReg<AArch64::ZT0, 0>(Node->getOperand(2), ZtValue))
2000 SDValue Ops[] = {ZtValue, Node->getOperand(3), Node->getOperand(4)};
2002 EVT VT = Node->getValueType(0);
2005 CurDAG->getMachineNode(Opc, DL, {MVT::Untyped, MVT::Other}, Ops);
2009 ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg(
2015 CurDAG->RemoveDeadNode(Node);
2021 EVT VT = N->getValueType(0);
2023 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
2025 SDValue Zn = N->getOperand(1 + NumVecs);
2026 SDValue Zm = N->getOperand(2 + NumVecs);
2030 SDNode *Intrinsic = CurDAG->getMachineNode(Op, DL, MVT::Untyped, Ops);
2033 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2036 CurDAG->RemoveDeadNode(N);
2071 TileNum = N->getConstantOperandVal(2);
2078 SliceBase = N->getOperand(2);
2080 SliceBase = N->getOperand(3);
2086 SDValue SubReg = CurDAG->getRegister(BaseReg, MVT::Other);
2087 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(0)};
2088 SDNode *Mov = CurDAG->getMachineNode(Op, DL, {MVT::Untyped, MVT::Other}, Ops);
2090 EVT VT = N->getValueType(0);
2093 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT,
2098 CurDAG->RemoveDeadNode(N);
2107 SDValue SliceBase = N->getOperand(2);
2109 SliceBase = N->getOperand(3);
2120 Ops.push_back(N->getOperand(2));
2123 Ops.push_back(N->getOperand(0)); //Chain
2124 SDNode *Mov = CurDAG->getMachineNode(Op, DL, {MVT::Untyped, MVT::Other}, Ops);
2126 EVT VT = N->getValueType(0);
2129 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT,
2135 CurDAG->RemoveDeadNode(N);
2143 EVT VT = N->getValueType(0);
2144 unsigned NumInVecs = N->getNumOperands() - 1;
2149 "Don't know how to handle multi-register input!");
2150 SmallVector<SDValue, 4> Regs(N->op_begin() + 1,
2151 N->op_begin() + 1 + NumInVecs);
2156 Ops.push_back(N->getOperand(1 + I));
2159 SDNode *Res = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops);
2163 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg(
2165 CurDAG->RemoveDeadNode(N);
2171 EVT VT = N->getOperand(2)->getValueType(0);
2175 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
2178 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)};
2179 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
2182 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2183 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2194 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
2201 N, Opc_rr, Opc_ri, N->getOperand(NumVecs + 3),
2202 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2204 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), // predicate
2207 N->getOperand(0)}; // chain
2208 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
2216 const DataLayout &DL = CurDAG->getDataLayout();
2221 int FI = FINode->getIndex();
2222 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
2223 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
2233 EVT VT = N->getOperand(2)->getValueType(0);
2239 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
2243 N->getOperand(NumVecs + 1), // base register
2244 N->getOperand(NumVecs + 2), // Incremental
2245 N->getOperand(0)}; // Chain
2246 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2252 /// WidenVector - Given a value in the V64 register class, produce the
2274 /// NarrowVector - Given a value in the V128 register class, produce the
2289 EVT VT = N->getValueType(0);
2293 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
2303 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 2);
2305 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2306 N->getOperand(NumVecs + 3), N->getOperand(0)};
2307 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2310 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
2314 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2321 CurDAG->RemoveDeadNode(N);
2327 EVT VT = N->getValueType(0);
2331 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
2340 RegSeq->getValueType(0), MVT::Other};
2342 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 1);
2345 CurDAG->getTargetConstant(LaneNo, dl,
2347 N->getOperand(NumVecs + 2), // Base register
2348 N->getOperand(NumVecs + 3), // Incremental
2349 N->getOperand(0)};
2350 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2361 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
2365 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
2375 CurDAG->RemoveDeadNode(N);
2381 EVT VT = N->getOperand(2)->getValueType(0);
2385 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
2393 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 2);
2395 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2396 N->getOperand(NumVecs + 3), N->getOperand(0)};
2397 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
2400 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2401 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2409 EVT VT = N->getOperand(2)->getValueType(0);
2413 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
2424 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 1);
2426 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2427 N->getOperand(NumVecs + 2), // Base Register
2428 N->getOperand(NumVecs + 3), // Incremental
2429 N->getOperand(0)};
2430 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2433 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2434 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2444 assert(N->getOpcode() == ISD::AND &&
2447 EVT VT = N->getValueType(0);
2455 // FIXME: simplify-demanded-bits in DAGCombine will probably have
2456 // changed the AND node to a 32-bit mask operation. We'll have to
2460 // from these situations when matching bigger pattern (bitfield insert).
2467 const SDNode *Op0 = N->getOperand(0).getNode();
2469 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
2480 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
2481 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL, SrlImm)) {
2482 // Extend the incoming operand of the SRL to 64-bit.
2483 Opd0 = Widen(CurDAG, Op0->getOperand(0).getOperand(0));
2487 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
2488 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL,
2491 Opd0 = Op0->getOperand(0).getOperand(0);
2494 VT = Opd0->getValueType(0);
2496 Opd0 = Op0->getOperand(0);
2501 // plus it may expose more opportunities for bitfield insert pattern.
2504 Opd0 = N->getOperand(0);
2520 : llvm::countr_one<uint64_t>(AndImm)) -
2536 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
2538 EVT VT = N->getValueType(0);
2543 SDValue Op = N->getOperand(0);
2544 if (Op->getOpcode() == ISD::TRUNCATE) {
2545 Op = Op->getOperand(0);
2546 VT = Op->getValueType(0);
2555 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2556 if (ShiftImm + Width > BitWidth)
2562 Imms = ShiftImm + Width - 1;
2576 // with MaskImm >> ShiftImm to search for the bit width.
2583 if (N->getOpcode() != ISD::SRL)
2587 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, AndMask))
2590 Opd0 = N->getOperand(0).getOperand(0);
2593 if (!isIntImmediate(N->getOperand(1), SrlImm))
2600 Opc = N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2609 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
2612 EVT VT = N->getValueType(0);
2627 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, ShlImm)) {
2628 Opd0 = N->getOperand(0).getOperand(0);
2629 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
2630 N->getOperand(0).getNode()->getOpcode() == ISD::TRUNCATE) {
2635 Opd0 = N->getOperand(0).getOperand(0);
2636 TruncBits = Opd0->getValueType(0).getSizeInBits() - VT.getSizeInBits();
2643 Opd0 = N->getOperand(0);
2657 if (!isIntImmediate(N->getOperand(1), SrlImm))
2662 int immr = SrlImm - ShlImm;
2664 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
2667 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
2669 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
2674 assert(N->getOpcode() == ISD::SIGN_EXTEND);
2676 EVT VT = N->getValueType(0);
2677 EVT NarrowVT = N->getOperand(0)->getValueType(0);
2682 SDValue Op = N->getOperand(0);
2687 // Extend the incoming operand of the shift to 64-bits.
2690 unsigned Imms = NarrowVT.getSizeInBits() - 1;
2691 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2692 CurDAG->getTargetConstant(Imms, dl, VT)};
2693 CurDAG->SelectNodeTo(N, AArch64::SBFMXri, VT, Ops);
2701 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
2704 switch (N->getOpcode()) {
2706 if (!N->isMachineOpcode())
2720 unsigned NOpc = N->getMachineOpcode();
2729 Opd0 = N->getOperand(0);
2730 Immr = N->getConstantOperandVal(1);
2731 Imms = N->getConstantOperandVal(2);
2744 EVT VT = N->getValueType(0);
2750 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
2751 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
2753 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
2754 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
2760 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
2761 CurDAG->getTargetConstant(Imms, dl, VT)};
2762 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2774 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
2803 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
2817 OpUsefulBits <<= MSB - Imm + 1;
2818 --OpUsefulBits;
2825 --OpUsefulBits;
2827 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
2830 OpUsefulBits.lshrInPlace(OpUsefulBits.getBitWidth() - Imm);
2839 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
2841 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2849 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2877 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2879 cast<const ConstantSDNode>(Op.getOperand(3).getNode())->getZExtValue();
2892 uint64_t Width = MSB - Imm + 1;
2895 OpUsefulBits <<= Width;
2896 --OpUsefulBits;
2909 uint64_t Width = MSB + 1;
2910 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
2912 OpUsefulBits <<= Width;
2913 --OpUsefulBits;
2934 if (!UserNode->isMachineOpcode())
2937 switch (UserNode->getMachineOpcode()) {
2953 if (UserNode->getOperand(0) != Orig && UserNode->getOperand(1) == Orig)
2963 if (UserNode->getOperand(0) != Orig)
2970 if (UserNode->getOperand(0) != Orig)
2989 for (SDNode *Node : Op.getNode()->uses()) {
3002 /// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3015 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3016 ShiftNode = CurDAG->getMachineNode(
3018 CurDAG->getTargetConstant(BitWidth - ShlAmount, dl, VT),
3019 CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, dl, VT));
3021 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3023 int ShrAmount = -ShlAmount;
3024 ShiftNode = CurDAG->getMachineNode(
3025 UBFMOpc, dl, VT, Op, CurDAG->getTargetConstant(ShrAmount, dl, VT),
3026 CurDAG->getTargetConstant(BitWidth - 1, dl, VT));
3032 // For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3037 int &Width);
3039 // For bit-field-positioning pattern "shl VAL, N)".
3044 int &Width);
3046 /// Does this tree qualify as an attempt to move a bitfield into position,
3050 int &DstLSB, int &Width) {
3056 KnownBits Known = CurDAG->computeKnownBits(Op);
3058 // Non-zero in the sense that they're not provably zero, which is the key
3069 NonZeroBits, Src, DstLSB, Width);
3072 NonZeroBits, Src, DstLSB, Width);
3082 int &Width) {
3108 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3113 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3134 Width = llvm::countr_one(NonZeroBits >> DstLSB);
3136 // Bail out on large Width. This happens when no proper combining / constant
3138 if (Width >= (int)VT.getSizeInBits()) {
3139 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3140 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3142 // If VT is i32, what Width >= 32 means:
3143 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3144 // demands at least 'Width' bits (after dag-combiner). This together with
3149 << "Found large Width in bit-field-positioning -- this indicates no "
3162 Src = getLeftShift(CurDAG, ShlOp0, ShlImm - DstLSB);
3170 int &Width) {
3192 Width = llvm::countr_one(ShiftedAndImm);
3204 int &Width) {
3219 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3223 Width = llvm::countr_one(NonZeroBits >> DstLSB);
3228 Src = getLeftShift(CurDAG, Op.getOperand(0), ShlImm - DstLSB);
3242 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3244 EVT VT = N->getValueType(0);
3261 SDValue And = N->getOperand(0);
3269 KnownBits Known = CurDAG->computeKnownBits(And);
3271 // Non-zero in the sense that they're not provably zero, which is the key
3286 // BFI/BFXIL dst, src, #lsb, #width.
3288 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3291 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3292 unsigned ImmS = Width - 1;
3302 // with a ORR-immediate with the zero register.
3317 SDNode *MOVI = CurDAG->getMachineNode(
3318 MOVIOpc, DL, VT, CurDAG->getTargetConstant(BFIImm, DL, VT));
3322 CurDAG->getTargetConstant(ImmR, DL, VT),
3323 CurDAG->getTargetConstant(ImmS, DL, VT)};
3325 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3332 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3371 SDNode *UBFMNode = CurDAG->getMachineNode(
3373 CurDAG->getTargetConstant(SrlImm + NumTrailingZeroInShiftedMask, DL,
3375 CurDAG->getTargetConstant(
3376 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3407 EVT VT = N->getValueType(0);
3408 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3409 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3410 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3429 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3438 CurDAG->getTargetConstant(EncodedShiftImm, DL, VT)};
3439 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3453 CurDAG->getTargetConstant(
3455 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3459 // Select the following pattern to left-shifted operand rather than BFI.
3472 CurDAG->getTargetConstant(
3474 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3481 // Select the following pattern to right-shifted operand rather than BFXIL.
3494 CurDAG->getTargetConstant(
3496 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3506 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3508 EVT VT = N->getValueType(0);
3514 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
3524 // countTrailingZeros(mask2) == imm2 - imm + 1
3543 SDValue OrOpd0Val = N->getOperand(I % 2);
3545 SDValue OrOpd1Val = N->getOperand((I + 1) % 2);
3549 int DstLSB, Width;
3558 // Compute the width of the bitfield insertion
3560 Width = ImmS - ImmR + 1;
3561 // FIXME: This constraint is to catch bitfield insertion we may
3564 if (Width <= 0)
3568 // can share the ImmR and ImmS values from the already-computed UBFM.
3571 Src, DstLSB, Width)) {
3572 ImmR = (BitWidth - DstLSB) % BitWidth;
3573 ImmS = Width - 1;
3583 // AND with imm. Indeed, simplify-demanded-bits may have removed
3585 KnownBits Known = CurDAG->computeKnownBits(OrOpd1Val);
3590 APInt::getBitsSet(Known.getBitWidth(), DstLSB, DstLSB + Width);
3600 Dst = OrOpd1->getOperand(0);
3602 // Maybe the AND has been removed by simplify-demanded-bits
3614 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(ImmR, DL, VT),
3615 CurDAG->getTargetConstant(ImmS, DL, VT)};
3617 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3625 SDValue And0 = N->getOperand(0);
3626 SDValue And1 = N->getOperand(1);
3641 SDValue Src = And1->getOperand(0);
3642 SDValue Dst = And0->getOperand(0);
3644 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
3646 // The BFXIL inserts the low-order bits from a source register, so right
3651 if (Src->hasOneUse() &&
3654 Src = Src->getOperand(0);
3658 SDNode *LSR = CurDAG->getMachineNode(
3659 ShiftOpc, DL, VT, Src, CurDAG->getTargetConstant(LsrImm, DL, VT),
3660 CurDAG->getTargetConstant(BitWidth - 1, DL, VT));
3663 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3664 unsigned ImmS = Width - 1;
3668 CurDAG->getTargetConstant(ImmR, DL, VT),
3669 CurDAG->getTargetConstant(ImmS, DL, VT)};
3671 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3679 if (N->getOpcode() != ISD::OR)
3687 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
3697 /// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
3701 if (N->getOpcode() != ISD::AND)
3704 EVT VT = N->getValueType(0);
3709 int DstLSB, Width;
3711 Op0, DstLSB, Width))
3715 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
3717 unsigned ImmS = Width - 1;
3720 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT),
3721 CurDAG->getTargetConstant(ImmS, DL, VT)};
3723 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3727 /// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
3730 EVT VT = N->getValueType(0);
3733 switch (N->getOpcode()) {
3761 SDValue ShiftAmt = N->getOperand(1);
3766 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
3767 ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
3768 ShiftAmt = ShiftAmt->getOperand(0);
3770 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
3771 SDValue Add0 = ShiftAmt->getOperand(0);
3772 SDValue Add1 = ShiftAmt->getOperand(1);
3776 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
3779 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
3782 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
3786 EVT SubVT = ShiftAmt->getValueType(0);
3796 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
3798 CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1);
3800 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
3801 isIntImmediate(Add0, Add0Imm) && (Add0Imm % Size == Size - 1)) {
3802 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
3806 EVT SubVT = ShiftAmt->getValueType(0);
3816 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
3818 CurDAG->getMachineNode(NotOpc, DL, SubVT, Zero, Add1);
3834 NewShiftAmt = ShiftAmt->getOperand(0);
3840 else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) {
3841 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32);
3842 MachineSDNode *Ext = CurDAG->getMachineNode(
3844 CurDAG->getTargetConstant(0, DL, MVT::i64), NewShiftAmt, SubReg);
3848 SDValue Ops[] = {N->getOperand(0), NewShiftAmt};
3849 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3859 FVal = CN->getValueAPF();
3862 if (LN->getOperand(1).getOpcode() != AArch64ISD::ADDlow ||
3863 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
3867 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
3868 FVal = cast<ConstantFP>(CN->getConstVal())->getValueAPF();
3873 // is between 1 and 32 for a destination w-register, or 1 and 64 for an
3874 // x-register.
3885 // fbits is between 1 and 64 in the worst-case, which means the fmul
3899 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
3924 return -1;
3939 "Unexpected non-integer value in special register string.");
3953 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
3954 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
3957 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
3960 int Imm = getIntOperandFromRegisterString(RegString->getString());
3961 if (Imm == -1) {
3965 AArch64SysReg::lookupSysRegByName(RegString->getString());
3966 if (TheReg && TheReg->Readable &&
3967 TheReg->haveFeatures(Subtarget->getFeatureBits()))
3968 Imm = TheReg->Encoding;
3970 Imm = AArch64SysReg::parseGenericRegister(RegString->getString());
3972 if (Imm == -1) {
3974 if (!ReadIs128Bit && RegString->getString() == "pc") {
3983 SDValue InChain = N->getOperand(0);
3984 SDValue SysRegImm = CurDAG->getTargetConstant(Imm, DL, MVT::i32);
3986 CurDAG->SelectNodeTo(N, Opcode64Bit, MVT::i64, MVT::Other /* Chain */,
3989 SDNode *MRRS = CurDAG->getMachineNode(
3996 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64, DL, MVT::i64,
3998 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64, DL, MVT::i64,
4014 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
4015 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4018 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4028 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4030 unsigned Reg = PMapper->Encoding;
4031 uint64_t Immed = N->getConstantOperandVal(2);
4032 CurDAG->SelectNodeTo(
4033 N, State, MVT::Other, CurDAG->getTargetConstant(Reg, DL, MVT::i32),
4034 CurDAG->getTargetConstant(Immed, DL, MVT::i16), N->getOperand(0));
4041 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4045 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4050 int Imm = getIntOperandFromRegisterString(RegString->getString());
4051 if (Imm == -1) {
4055 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
4056 if (TheReg && TheReg->Writeable &&
4057 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4058 Imm = TheReg->Encoding;
4060 Imm = AArch64SysReg::parseGenericRegister(RegString->getString());
4062 if (Imm == -1)
4066 SDValue InChain = N->getOperand(0);
4068 CurDAG->SelectNodeTo(N, AArch64::MSR, MVT::Other,
4069 CurDAG->getTargetConstant(Imm, DL, MVT::i32),
4070 N->getOperand(2), InChain);
4074 SDNode *Pair = CurDAG->getMachineNode(
4076 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(), DL,
4078 N->getOperand(2),
4079 CurDAG->getTargetConstant(AArch64::sube64, DL, MVT::i32),
4080 N->getOperand(3),
4081 CurDAG->getTargetConstant(AArch64::subo64, DL, MVT::i32)});
4083 CurDAG->SelectNodeTo(N, AArch64::MSRR, MVT::Other,
4084 CurDAG->getTargetConstant(Imm, DL, MVT::i32),
4091 /// We've got special pseudo-instructions for these
4094 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
4097 if (Subtarget->hasLSE()) return false;
4111 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
4112 N->getOperand(0)};
4113 SDNode *CmpSwap = CurDAG->getMachineNode(
4115 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4117 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4118 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
4122 CurDAG->RemoveDeadNode(N);
4134 ->getAPIntValue()
4141 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4142 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
4149 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4150 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
4155 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
4156 Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32);
4175 ->getAPIntValue()
4180 Val = -Val;
4184 // means we can only use the immediate form when the operand is non-negative.
4191 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4192 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
4199 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4200 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
4205 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
4206 Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32);
4224 ->getAPIntValue()
4231 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4232 Imm = CurDAG->getTargetConstant(Val & 0xFF, DL, MVT::i32);
4238 if (Val >= -128 && Val <= 127) {
4239 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4240 Imm = CurDAG->getTargetConstant(Val & 0xFF, DL, MVT::i32);
4244 if (Val >= -32768 && Val <= 32512 && Val % 256 == 0) {
4245 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
4246 Imm = CurDAG->getTargetConstant((Val >> 8) & 0xFF, DL, MVT::i32);
4259 int64_t ImmVal = CNode->getSExtValue();
4261 if (ImmVal >= -128 && ImmVal < 128) {
4262 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
4271 uint64_t ImmVal = CNode->getZExtValue();
4290 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32);
4300 uint64_t ImmVal = CNode->getZExtValue();
4331 Imm = CurDAG->getTargetConstant(encoding, DL, MVT::i64);
4347 uint64_t ImmVal = CN->getZExtValue();
4360 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32);
4369 // since the offset between FrameIndex and IRGstack is a compile-time
4371 if (!(isa<FrameIndexSDNode>(N->getOperand(1)))) {
4375 SDValue IRG_SP = N->getOperand(2);
4376 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4377 IRG_SP->getConstantOperandVal(1) != Intrinsic::aarch64_irg_sp) {
4383 int FI = cast<FrameIndexSDNode>(N->getOperand(1))->getIndex();
4384 SDValue FiOp = CurDAG->getTargetFrameIndex(
4385 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
4386 int TagOffset = N->getConstantOperandVal(3);
4388 SDNode *Out = CurDAG->getMachineNode(
4390 {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2),
4391 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)});
4397 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4402 // compile-time constant, not just for stack allocations.
4406 int TagOffset = N->getConstantOperandVal(3);
4407 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64,
4408 {N->getOperand(1), N->getOperand(2)});
4409 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64,
4410 {SDValue(N1, 0), N->getOperand(2)});
4411 SDNode *N3 = CurDAG->getMachineNode(
4413 {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64),
4414 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)});
4419 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4422 if (N->getConstantOperandVal(2) != 0)
4424 if (!N->getOperand(0).isUndef())
4428 EVT VT = N->getValueType(0);
4429 EVT InVT = N->getOperand(1).getValueType();
4443 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64);
4444 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT,
4445 N->getOperand(1), RC));
4450 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4453 if (N->getConstantOperandVal(1) != 0)
4457 EVT VT = N->getValueType(0);
4458 EVT InVT = N->getOperand(0).getValueType();
4472 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64);
4473 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT,
4474 N->getOperand(0), RC));
4479 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4481 SDValue N0 = N->getOperand(0);
4482 SDValue N1 = N->getOperand(1);
4483 EVT VT = N->getValueType(0);
4485 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4489 // OR N0, N1 -> xar (x, y, imm)
4491 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4492 // N0 = SHL_PRED true, V, splat(bits-imm)
4495 (Subtarget->hasSVE2() ||
4496 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4505 if (!TLI->isAllActivePredicate(*CurDAG, N0.getOperand(0)) ||
4506 !TLI->isAllActivePredicate(*CurDAG, N1.getOperand(0)))
4523 CurDAG->getTargetConstant(ShrAmt.getZExtValue(), DL, MVT::i32);
4529 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
4535 if (!Subtarget->hasSHA3())
4538 if (N0->getOpcode() != AArch64ISD::VSHL ||
4539 N1->getOpcode() != AArch64ISD::VLSHR)
4542 if (N0->getOperand(0) != N1->getOperand(0) ||
4543 N1->getOperand(0)->getOpcode() != ISD::XOR)
4554 SDValue Imm = CurDAG->getTargetConstant(
4561 CurDAG->SelectNodeTo(N, AArch64::XAR, N0.getValueType(), Ops);
4568 if (Node->isMachineOpcode()) {
4569 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
4570 Node->setNodeId(-1);
4575 EVT VT = Node->getValueType(0);
4577 switch (Node->getOpcode()) {
4649 if (ConstNode->isZero()) {
4651 SDValue New = CurDAG->getCopyFromReg(
4652 CurDAG->getEntryNode(), SDLoc(Node), AArch64::WZR, MVT::i32);
4656 SDValue New = CurDAG->getCopyFromReg(
4657 CurDAG->getEntryNode(), SDLoc(Node), AArch64::XZR, MVT::i64);
4667 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
4670 SDValue TFI = CurDAG->getTargetFrameIndex(
4671 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
4673 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
4674 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) };
4675 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops);
4679 unsigned IntNo = Node->getConstantOperandVal(1);
4685 SDValue Chain = Node->getOperand(0);
4686 SDValue Val = Node->getOperand(2);
4687 SDValue Zero = CurDAG->getCopyFromReg(Chain, DL, AArch64::XZR, MVT::i64);
4689 CurDAG->getMachineNode(AArch64::GCSSS1, DL, MVT::Other, Val, Chain);
4690 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2, DL, MVT::i64,
4699 SDValue MemAddr = Node->getOperand(2);
4701 SDValue Chain = Node->getOperand(0);
4703 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64,
4708 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
4709 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
4718 SDValue Chain = Node->getOperand(0);
4719 SDValue ValLo = Node->getOperand(2);
4720 SDValue ValHi = Node->getOperand(3);
4721 SDValue MemAddr = Node->getOperand(4);
4726 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops);
4729 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
4730 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
5070 if (Subtarget->hasSME2())
5073 else if (Subtarget->hasSVE2p1())
5081 if (Subtarget->hasSME2())
5084 else if (Subtarget->hasSVE2p1())
5091 if (Subtarget->hasSME2())
5094 else if (Subtarget->hasSVE2p1())
5101 if (Subtarget->hasSME2())
5104 else if (Subtarget->hasSVE2p1())
5115 if (Subtarget->hasSME2())
5118 else if (Subtarget->hasSVE2p1())
5126 if (Subtarget->hasSME2())
5129 else if (Subtarget->hasSVE2p1())
5136 if (Subtarget->hasSME2())
5139 else if (Subtarget->hasSVE2p1())
5146 if (Subtarget->hasSME2())
5149 else if (Subtarget->hasSVE2p1())
5160 if (Subtarget->hasSME2())
5164 else if (Subtarget->hasSVE2p1())
5172 if (Subtarget->hasSME2())
5176 else if (Subtarget->hasSVE2p1())
5183 if (Subtarget->hasSME2())
5187 else if (Subtarget->hasSVE2p1())
5194 if (Subtarget->hasSME2())
5198 else if (Subtarget->hasSVE2p1())
5209 if (Subtarget->hasSME2())
5213 else if (Subtarget->hasSVE2p1())
5221 if (Subtarget->hasSME2())
5225 else if (Subtarget->hasSVE2p1())
5232 if (Subtarget->hasSME2())
5236 else if (Subtarget->hasSVE2p1())
5243 if (Subtarget->hasSME2())
5247 else if (Subtarget->hasSVE2p1())
5472 SDValue Chain = Node->getOperand(0);
5473 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, DL, AArch64::FP, MVT::i64);
5475 CurDAG->getMachineNode(AArch64::SUBXri, DL, MVT::i64, CopyFP,
5476 CurDAG->getTargetConstant(8, DL, MVT::i32),
5477 CurDAG->getTargetConstant(0, DL, MVT::i32)),
5481 CurDAG->RemoveDeadNode(Node);
5483 auto &MF = CurDAG->getMachineFunction();
5485 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
5490 Node->getValueType(0),
5499 Node->getValueType(0),
5507 Node->getValueType(0),
5516 Node->getValueType(0),
5526 unsigned IntNo = Node->getConstantOperandVal(0);
5574 Node->getValueType(0),
5581 Node->getValueType(0),
5588 Node->getValueType(0),
5595 Node->getValueType(0),
5602 Node->getValueType(0),
5609 Node->getValueType(0),
5616 Node->getValueType(0),
5623 Node->getValueType(0),
5630 Node->getValueType(0),
5637 Node->getValueType(0),
5644 Node->getValueType(0),
5651 Node->getValueType(0),
5658 Node->getValueType(0),
5665 Node->getValueType(0),
5672 Node->getValueType(0),
5679 Node->getValueType(0),
5686 Node->getValueType(0),
5693 Node->getValueType(0),
5700 Node->getValueType(0),
5707 Node->getValueType(0),
5714 Node->getValueType(0),
5721 Node->getValueType(0),
5728 Node->getValueType(0),
5735 Node->getValueType(0),
5742 Node->getValueType(0),
5749 Node->getValueType(0),
5756 Node->getValueType(0),
5763 Node->getValueType(0),
5770 Node->getValueType(0),
5777 Node->getValueType(0),
5784 Node->getValueType(0),
5791 Node->getValueType(0),
5798 Node->getValueType(0),
5805 Node->getValueType(0),
5812 Node->getValueType(0),
5819 Node->getValueType(0),
5826 Node->getValueType(0),
5833 Node->getValueType(0),
5840 Node->getValueType(0),
5847 Node->getValueType(0),
5854 Node->getValueType(0),
5861 Node->getValueType(0),
5868 Node->getValueType(0),
5875 Node->getValueType(0),
5882 Node->getValueType(0),
5889 Node->getValueType(0),
5896 Node->getValueType(0),
5903 Node->getValueType(0),
5910 Node->getValueType(0),
5917 Node->getValueType(0),
5924 Node->getValueType(0),
5931 Node->getValueType(0),
5968 Node->getValueType(0),
5975 Node->getValueType(0),
5982 Node->getValueType(0),
5992 Node->getValueType(0),
5999 Node->getValueType(0),
6006 Node->getValueType(0),
6016 Node->getValueType(0),
6023 Node->getValueType(0),
6030 Node->getValueType(0),
6041 Node->getValueType(0),
6052 Node->getValueType(0),
6063 Node->getValueType(0),
6074 Node->getValueType(0),
6081 Node->getValueType(0),
6112 Node->getValueType(0),
6119 Node->getValueType(0),
6126 Node->getValueType(0),
6133 Node->getValueType(0),
6140 Node->getValueType(0),
6150 unsigned IntNo = Node->getConstantOperandVal(1);
6151 if (Node->getNumOperands() >= 3)
6152 VT = Node->getOperand(2)->getValueType(0);
6816 VT = Node->getOperand(1).getValueType();
6845 VT = Node->getOperand(1).getValueType();
6874 VT = Node->getOperand(1).getValueType();
6903 VT = Node->getOperand(1).getValueType();
6932 VT = Node->getOperand(1).getValueType();
6961 VT = Node->getOperand(1).getValueType();
6990 VT = Node->getOperand(1).getValueType();
7010 VT = Node->getOperand(1).getValueType();
7030 VT = Node->getOperand(1).getValueType();
7106 /// createAArch64ISelDag - This pass converts a legalized DAG into a
7107 /// AArch64-specific DAG, ready for instruction scheduling.
7142 return cast<MemSDNode>(Root)->getMemoryVT();
7145 return cast<MemIntrinsicSDNode>(Root)->getMemoryVT();
7147 const unsigned Opcode = Root->getOpcode();
7155 return cast<VTSDNode>(Root->getOperand(3))->getVT();
7157 return cast<VTSDNode>(Root->getOperand(4))->getVT();
7160 Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/2);
7163 Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/3);
7166 Ctx, Root->getOperand(1)->getValueType(0), /*NumVec=*/4);
7174 switch (Root->getConstantOperandVal(1)) {
7182 // width of the predicate.
7184 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1);
7188 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/2);
7191 Ctx, Root->getOperand(4)->getValueType(0), /*NumVec=*/2);
7195 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/3);
7198 Ctx, Root->getOperand(5)->getValueType(0), /*NumVec=*/3);
7202 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/4);
7205 Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4);
7215 /// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7222 const EVT MemVT = getMemVTFromNode(*(CurDAG->getContext()), Root);
7223 const DataLayout &DL = CurDAG->getDataLayout();
7224 const MachineFrameInfo &MFI = MF->getFrameInfo();
7227 int FI = cast<FrameIndexSDNode>(N)->getIndex();
7231 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
7232 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
7251 int64_t MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
7262 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
7266 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
7269 OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64);
7294 int64_t ImmOff = C->getSExtValue();
7304 Offset = CurDAG->getTargetConstant(ImmOff >> Scale, DL, MVT::i64);
7306 SDNode *MI = CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
7317 if (C->getZExtValue() == Scale) {
7330 return TLI->isAllActivePredicate(*CurDAG, N);
7344 int64_t ImmOff = C->getSExtValue();
7347 Offset = CurDAG->getTargetConstant(ImmOff / Scale, SDLoc(N), MVT::i64);
7354 Offset = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);