Lines Matching defs:AArch64
1 //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
120 cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
125 cl::desc("Enable AArch64 logical imm instruction "
135 cl::desc("Combine extends of AArch64 masked "
161 static const MCPhysReg GPRArgRegs[] = {AArch64::X0, AArch64::X1, AArch64::X2,
162 AArch64::X3, AArch64::X4, AArch64::X5,
163 AArch64::X6, AArch64::X7};
164 static const MCPhysReg FPRArgRegs[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2,
165 AArch64::Q3, AArch64::Q4, AArch64::Q5,
166 AArch64::Q6, AArch64::Q7};
168 ArrayRef<MCPhysReg> llvm::AArch64::getGPRArgRegs() { return GPRArgRegs; }
170 ArrayRef<MCPhysReg> llvm::AArch64::getFPRArgRegs() { return FPRArgRegs; }
238 VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock;
369 AddrDisc = DAG->getRegister(AArch64::NoRegister, MVT::i64);
379 // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
387 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
388 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
391 addRegisterClass(MVT::i64x8, &AArch64::GPR64x8ClassRegClass);
397 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
398 addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
399 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
400 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
401 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
405 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
406 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
429 addRegisterClass(MVT::nxv1i1, &AArch64::PPRRegClass);
430 addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
431 addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
432 addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
433 addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
436 addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
437 addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
438 addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
439 addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
441 addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
442 addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
443 addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
444 addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
445 addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
446 addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
448 addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
449 addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
450 addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
455 addRegisterClass(VT, &AArch64::ZPRRegClass);
459 addRegisterClass(VT, &AArch64::ZPRRegClass);
464 addRegisterClass(MVT::aarch64svcount, &AArch64::PPRRegClass);
630 // AArch64 lacks both left-rotate and popcount instructions.
638 // AArch64 doesn't have i32 MULH{S|U}.
642 // AArch64 doesn't have {U|S}MUL_LOHI.
862 // AArch64 has implementations of a lot of rounding-like FP operations.
1026 // AArch64 does not have floating-point extending loads, i1 sign-extending
1148 setStackPointerRegisterToSaveRestore(AArch64::SP);
1211 // AArch64 doesn't have a direct vector ->f32 conversion instructions for
1237 // when AArch64 doesn't have fullfp16 support, promote the input
1347 // AArch64 has implementations of a lot of rounding-like FP operations.
2128 addRegisterClass(VT, &AArch64::FPR64RegClass);
2134 addRegisterClass(VT, &AArch64::FPR128RegClass);
2289 NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
2292 NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
2295 NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
2524 return AArch64::createFastISel(funcInfo, libInfo);
2899 BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
2900 BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
2908 TrueBB->addLiveIn(AArch64::NZCV);
2909 EndBB->addLiveIn(AArch64::NZCV);
2912 BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
2968 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::LDR_ZA));
2970 MIB.addReg(AArch64::ZA, RegState::Define);
3004 bool HasTile = BaseReg != AArch64::ZA;
3034 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::ZERO_M));
3040 MIB.addDef(AArch64::ZAD0 + I, RegState::ImplicitDefine);
3057 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRXui))
3062 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRHHui))
3063 .addReg(AArch64::WZR)
3066 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRWui))
3067 .addReg(AArch64::WZR)
3098 Register SP = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3100 .addReg(AArch64::SP);
3105 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::MSUBXrrr), Dest)
3110 AArch64::SP)
3124 int SMEOrigInstr = AArch64::getSMEPseudoMap(MI.getOpcode());
3128 TII->get(MI.getOpcode()).TSFlags & AArch64::SMEMatrixTypeMask;
3130 case (AArch64::SMEMatrixArray):
3131 return EmitZAInstr(SMEOrigInstr, AArch64::ZA, MI, BB);
3132 case (AArch64::SMEMatrixTileB):
3133 return EmitZAInstr(SMEOrigInstr, AArch64::ZAB0, MI, BB);
3134 case (AArch64::SMEMatrixTileH):
3135 return EmitZAInstr(SMEOrigInstr, AArch64::ZAH0, MI, BB);
3136 case (AArch64::SMEMatrixTileS):
3137 return EmitZAInstr(SMEOrigInstr, AArch64::ZAS0, MI, BB);
3138 case (AArch64::SMEMatrixTileD):
3139 return EmitZAInstr(SMEOrigInstr, AArch64::ZAD0, MI, BB);
3140 case (AArch64::SMEMatrixTileQ):
3141 return EmitZAInstr(SMEOrigInstr, AArch64::ZAQ0, MI, BB);
3151 case AArch64::InitTPIDR2Obj:
3153 case AArch64::AllocateZABuffer:
3155 case AArch64::F128CSEL:
3165 AArch64::LR, /*isDef*/ true,
3177 case AArch64::CATCHRET:
3180 case AArch64::PROBED_STACKALLOC_DYN:
3183 case AArch64::LD1_MXIPXX_H_PSEUDO_B:
3184 return EmitTileLoad(AArch64::LD1_MXIPXX_H_B, AArch64::ZAB0, MI, BB);
3185 case AArch64::LD1_MXIPXX_H_PSEUDO_H:
3186 return EmitTileLoad(AArch64::LD1_MXIPXX_H_H, AArch64::ZAH0, MI, BB);
3187 case AArch64::LD1_MXIPXX_H_PSEUDO_S:
3188 return EmitTileLoad(AArch64::LD1_MXIPXX_H_S, AArch64::ZAS0, MI, BB);
3189 case AArch64::LD1_MXIPXX_H_PSEUDO_D:
3190 return EmitTileLoad(AArch64::LD1_MXIPXX_H_D, AArch64::ZAD0, MI, BB);
3191 case AArch64::LD1_MXIPXX_H_PSEUDO_Q:
3192 return EmitTileLoad(AArch64::LD1_MXIPXX_H_Q, AArch64::ZAQ0, MI, BB);
3193 case AArch64::LD1_MXIPXX_V_PSEUDO_B:
3194 return EmitTileLoad(AArch64::LD1_MXIPXX_V_B, AArch64::ZAB0, MI, BB);
3195 case AArch64::LD1_MXIPXX_V_PSEUDO_H:
3196 return EmitTileLoad(AArch64::LD1_MXIPXX_V_H, AArch64::ZAH0, MI, BB);
3197 case AArch64::LD1_MXIPXX_V_PSEUDO_S:
3198 return EmitTileLoad(AArch64::LD1_MXIPXX_V_S, AArch64::ZAS0, MI, BB);
3199 case AArch64::LD1_MXIPXX_V_PSEUDO_D:
3200 return EmitTileLoad(AArch64::LD1_MXIPXX_V_D, AArch64::ZAD0, MI, BB);
3201 case AArch64::LD1_MXIPXX_V_PSEUDO_Q:
3202 return EmitTileLoad(AArch64::LD1_MXIPXX_V_Q, AArch64::ZAQ0, MI, BB);
3203 case AArch64::LDR_ZA_PSEUDO:
3205 case AArch64::LDR_TX_PSEUDO:
3206 return EmitZTInstr(MI, BB, AArch64::LDR_TX, /*Op0IsDef=*/true);
3207 case AArch64::STR_TX_PSEUDO:
3208 return EmitZTInstr(MI, BB, AArch64::STR_TX, /*Op0IsDef=*/false);
3209 case AArch64::ZERO_M_PSEUDO:
3211 case AArch64::ZERO_T_PSEUDO:
3212 return EmitZTInstr(MI, BB, AArch64::ZERO_T, /*Op0IsDef=*/true);
3217 // AArch64 Lowering private implementation.
3250 /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
3279 /// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
3340 /// Convert a DAG fp condition code to an AArch64 CC.
3369 /// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
3898 // LHS one, the extreme case being when RHS is an immediate. However, AArch64
4057 // Emit the AArch64 operation with overflow check.
4349 SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow);
4489 // AArch64 FP-to-int conversions saturate to the destination element size, so
4567 // AArch64 FP-to-int conversions saturate to the destination register size, so
4946 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, OpVT, Op);
5134 DAG.getConstant(AArch64::RoundingBitsPos, DL, MVT::i32));
5146 const int RMMask = ~(AArch64::Rounding::rmMask << AArch64::RoundingBitsPos);
5206 DAG.getConstant(AArch64::ReservedFPControlBits, DL, MVT::i64));
5637 report_fatal_error("Unexpected type for AArch64 NEON intrinic");
6513 assert((Subtarget->hasFeature(AArch64::FeatureLSE2) &&
6514 Subtarget->hasFeature(AArch64::FeatureRCPC3) && IsStoreRelease) ||
7353 RC = &AArch64::GPR32RegClass;
7355 RC = &AArch64::GPR64RegClass;
7357 RC = &AArch64::FPR16RegClass;
7359 RC = &AArch64::FPR32RegClass;
7361 RC = &AArch64::FPR64RegClass;
7363 RC = &AArch64::FPR128RegClass;
7367 RC = &AArch64::PPRRegClass;
7370 RC = &AArch64::PPRRegClass;
7373 RC = &AArch64::ZPRRegClass;
7451 Register VReg = MF.addLiveIn(AArch64::X4, &AArch64::GPR64RegClass);
7615 if (!CCInfo.isAllocated(AArch64::X8)) {
7616 Register X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
7617 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
7722 auto GPRArgRegs = AArch64::getGPRArgRegs();
7745 // compute its address relative to x4. For a normal AArch64->AArch64
7748 Register VReg = MF.addLiveIn(AArch64::X4, &AArch64::GPR64RegClass);
7757 Register VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
7773 auto FPRArgRegs = AArch64::getFPRArgRegs();
7785 Register VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
8020 // tail-called on AArch64 when the OS does not support dynamic
8166 if (MI.getOpcode() == AArch64::MSRpstatesvcrImm1 ||
8167 MI.getOpcode() == AArch64::MSRpstatePseudo) {
8171 (AArch64::GPR32RegClass.contains(MO.getReg()) ||
8172 AArch64::GPR64RegClass.contains(MO.getReg())))
8178 MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/false,
8180 MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/true,
8192 (MI.getOpcode() == AArch64::ADDXri ||
8193 MI.getOpcode() == AArch64::SUBXri)) {
8197 MI.addOperand(MachineOperand::CreateReg(AArch64::VG, /*IsDef=*/false,
8303 return AArch64::ZPRRegClass.contains(Loc.getLocReg()) ||
8304 AArch64::PPRRegClass.contains(Loc.getLocReg());
8471 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
8725 RegsToPass.emplace_back(AArch64::X4, ParamPtr);
8726 RegsToPass.emplace_back(AArch64::X5,
8956 Result = DAG.getCopyToReg(Result, DL, AArch64::X0, TPIDR2Block, Glue);
8959 {Result, TPIDR2_EL0, DAG.getRegister(AArch64::X0, MVT::i64),
9104 // Windows AArch64 ABIs require that for returning structs by value we copy
9112 unsigned RetValReg = AArch64::X0;
9114 RetValReg = AArch64::X8;
9125 if (AArch64::GPR64RegClass.contains(*I))
9127 else if (AArch64::FPR64RegClass.contains(*I))
9349 // normal AArch64 call node: x0 takes the address of the descriptor, and
9351 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
9363 Ops.push_back(DAG.getRegister(AArch64::NoRegister, MVT::i64)); // Addr Disc.
9366 Ops.push_back(DAG.getRegister(AArch64::X0, MVT::i64));
9370 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
9392 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
9407 Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
9411 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
9427 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
9430 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
9450 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
9453 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
9456 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
9494 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
9563 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
9566 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
9593 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
9636 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
9706 return SDValue(DAG.getMachineNode(AArch64::LOADauthptrstatic, DL, MVT::i64,
9760 : DAG.getRegister(AArch64::XZR, MVT::i64);
9766 DAG.getMachineNode(AArch64::MOVaddrPAC, DL, MVT::i64,
9775 DAG.getMachineNode(AArch64::LOADgotPAC, DL, MVT::i64,
9921 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
9995 SetVecVal(AArch64::dsub);
9998 SetVecVal(AArch64::ssub);
10001 SetVecVal(AArch64::hsub);
10022 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, BSP);
10024 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, BSP);
10026 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, BSP);
10381 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
10615 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
10685 unsigned BlockSize = AArch64::SVEBitsPerBlock / Ty.getVectorMinNumElements();
10768 TVal = DAG.getTargetInsertSubreg(AArch64::hsub, DL, MVT::f32,
10770 FVal = DAG.getTargetInsertSubreg(AArch64::hsub, DL, MVT::f32,
10777 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, Ty, Res);
10826 SDValue X16Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::X16,
10828 SDNode *B = DAG.getMachineNode(AArch64::BR_JumpTable, DL, MVT::Other,
10835 DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
10848 Dest->getMachineOpcode() == AArch64::JumpTableDest32)
10861 SDValue AddrDisc = DAG.getRegister(AArch64::XZR, MVT::i64);
10863 SDNode *BrA = DAG.getMachineNode(AArch64::BRA, DL, MVT::Other,
10901 SDValue AddrDisc = DAG.getRegister(AArch64::XZR, MVT::i64);
10904 DAG.getMachineNode(AArch64::MOVaddrPAC, DL, {MVT::Other, MVT::Glue},
10906 return DAG.getCopyFromReg(SDValue(MOV, 0), DL, AArch64::X16, MVT::i64,
10943 // relative to x4. For a normal AArch64->AArch64 call, x4 == sp on entry,
10945 Register VReg = MF.addLiveIn(AArch64::X4, &AArch64::GPR64RegClass);
10967 // The layout of the va_list struct is specified in the AArch64 Procedure Call
11156 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
11186 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
11207 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
11232 Register Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
11242 St = DAG.getMachineNode(AArch64::XPACI, DL, VT, ReturnAddress);
11246 DAG.getCopyToReg(DAG.getEntryNode(), DL, AArch64::LR, ReturnAddress);
11247 St = DAG.getMachineNode(AArch64::XPACLRI, DL, VT, Chain);
11312 // AArch64 Optimization Hooks
11377 // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
11407 // AArch64 reciprocal iteration instruction: (2 - M * N)
11422 // AArch64 Inline Assembly Support
11489 return VT == MVT::aarch64svcount ? &AArch64::PNR_p8to15RegClass
11490 : &AArch64::PPR_p8to15RegClass;
11492 return VT == MVT::aarch64svcount ? &AArch64::PNR_3bRegClass
11493 : &AArch64::PPR_3bRegClass;
11495 return VT == MVT::aarch64svcount ? &AArch64::PNRRegClass
11496 : &AArch64::PPRRegClass;
11519 return &AArch64::MatrixIndexGPR32_8_11RegClass;
11521 return &AArch64::MatrixIndexGPR32_12_15RegClass;
11575 Glue = DAG.getCopyFromReg(Chain, DL, AArch64::NZCV, MVT::i32, Glue);
11578 Glue = DAG.getCopyFromReg(Chain, DL, AArch64::NZCV, MVT::i32);
11676 return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
11678 return std::make_pair(0U, &AArch64::GPR64commonRegClass);
11679 return std::make_pair(0U, &AArch64::GPR32commonRegClass);
11685 return std::make_pair(0U, &AArch64::ZPRRegClass);
11692 return std::make_pair(0U, &AArch64::FPR16RegClass);
11694 return std::make_pair(0U, &AArch64::FPR32RegClass);
11696 return std::make_pair(0U, &AArch64::FPR64RegClass);
11698 return std::make_pair(0U, &AArch64::FPR128RegClass);
11707 return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
11709 return std::make_pair(0U, &AArch64::FPR128_loRegClass);
11715 return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
11729 return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
11732 return std::make_pair(unsigned(AArch64::ZA), &AArch64::MPRRegClass);
11736 return std::make_pair(unsigned(AArch64::ZT0), &AArch64::ZTRRegClass);
11756 Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
11757 Res.second = &AArch64::FPR64RegClass;
11759 Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
11760 Res.second = &AArch64::FPR128RegClass;
11767 !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
11768 !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
11807 Result = DAG.getRegister(AArch64::XZR, MVT::i64);
11809 Result = DAG.getRegister(AArch64::WZR, MVT::i32);
11927 // AArch64 Advanced SIMD Support
13353 if (VT.getSizeInBits().getKnownMinValue() != AArch64::SVEBitsPerBlock)
13671 unsigned VScale = MaxSVESize / AArch64::SVEBitsPerBlock;
15131 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
15377 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
15383 Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
15401 Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
15404 Chain, Callee, DAG.getRegister(AArch64::X15, MVT::i64),
15414 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
15420 Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
15442 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
17343 // AArch64 has five basic addressing modes:
17478 AArch64::X16, AArch64::X17, AArch64::LR, 0
17484 static const MCPhysReg RCRegs[] = {AArch64::FPCR};
19325 // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8.
19644 // AArch64 high-vector "long" operations are formed by performing the non-high
19718 /// Helper structure to keep track of a SET_CC lowered into AArch64 code.
19727 AArch64SetCCInfo AArch64;
19740 /// AArch64 lowered one.
19761 SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3);
19763 SetCCInfo.Info.AArch64.CC =
19780 SetCCInfo.Info.AArch64.CC =
19781 AArch64CC::getInvertedCondCode(SetCCInfo.Info.AArch64.CC);
19822 ? InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
19832 AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
19834 Cmp = *InfoAndKind.Info.AArch64.Cmp;
20355 // Bail out when value type is not one of {i32, i64}, since AArch64 ADD with
20375 // On many AArch64 processors (Cortex A78, Neoverse N1/N2/V1, etc), ADD with
20975 if (VT.getSizeInBits().getKnownMinValue() != AArch64::SVEBitsPerBlock)
21688 if (VT.getSizeInBits().getKnownMinValue() > AArch64::SVEBitsPerBlock)
21866 ZeroReg = AArch64::WZR;
21869 ZeroReg = AArch64::XZR;
22495 Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
23132 Subtarget.getMaxSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
23281 Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
23418 // when it is safe to remove the AND. Unfortunately it only runs on AArch64 and
24514 if (SrcVT.getSizeInBits().getKnownMinValue() > AArch64::SVEBitsPerBlock)
24625 if (RetVT.getSizeInBits().getKnownMinValue() > AArch64::SVEBitsPerBlock)
25094 // You can see the regressions on test/CodeGen/AArch64/aarch64-smull.ll
25159 // You can see the regressions on test/CodeGen/AArch64/aarch64-smull.ll
25845 Op = DAG.getTargetInsertSubreg(AArch64::hsub, DL, MVT::f32,
25954 DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, dl, MVT::i32);
25955 SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
25956 SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
25983 Opcode = AArch64::CASPX;
25986 Opcode = AArch64::CASPAX;
25989 Opcode = AArch64::CASPLX;
25993 Opcode = AArch64::CASPALX;
26003 unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64;
26019 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
26022 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
26025 Opcode = AArch64::CMP_SWAP_128_RELEASE;
26029 Opcode = AArch64::CMP_SWAP_128;
26065 return AArch64::LDCLRP;
26068 return AArch64::LDCLRPA;
26071 return AArch64::LDCLRPL;
26075 return AArch64::LDCLRPAL;
26085 return AArch64::LDSETP;
26088 return AArch64::LDSETPA;
26091 return AArch64::LDSETPL;
26095 return AArch64::LDSETPAL;
26105 return AArch64::SWPP;
26108 return AArch64::SWPPA;
26111 return AArch64::SWPPL;
26115 return AArch64::SWPPAL;
26287 // Non-volatile or atomic loads are optimized later in AArch64's load/store
26299 assert(Subtarget->hasFeature(AArch64::FeatureRCPC3));
26880 if (AArch64::GPR64RegClass.contains(*I))
26881 RC = &AArch64::GPR64RegClass;
26882 else if (AArch64::FPR64RegClass.contains(*I))
26883 RC = &AArch64::FPR64RegClass;
26909 // Integer division on AArch64 is expensive. However, when aggressively
26912 // The exception to this is vector division. Since AArch64 doesn't have vector
26944 case AArch64::BLR:
26945 case AArch64::BLRNoIP:
26946 case AArch64::TCRETURNri:
26947 case AArch64::TCRETURNrix16x17:
26948 case AArch64::TCRETURNrix17:
26949 case AArch64::TCRETURNrinotx16:
26959 return BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(AArch64::KCFI_CHECK))
27073 case AArch64::ADRP:
27074 case AArch64::G_ADD_LOW:
28152 unsigned MinNumElts = AArch64::SVEBitsPerBlock / BitsPerElt;
28279 // equivalent to an AArch64 instruction. There's the extra component of
28446 MaxSVEVectorSizeInBits = AArch64::SVEMaxBitsPerVector;