1 //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the AArch64 implementation of the TargetRegisterInfo 10 // class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64RegisterInfo.h" 15 #include "AArch64FrameLowering.h" 16 #include "AArch64InstrInfo.h" 17 #include "AArch64MachineFunctionInfo.h" 18 #include "AArch64Subtarget.h" 19 #include "MCTargetDesc/AArch64AddressingModes.h" 20 #include "MCTargetDesc/AArch64InstPrinter.h" 21 #include "llvm/ADT/BitVector.h" 22 #include "llvm/BinaryFormat/Dwarf.h" 23 #include "llvm/CodeGen/MachineFrameInfo.h" 24 #include "llvm/CodeGen/MachineInstrBuilder.h" 25 #include "llvm/CodeGen/MachineRegisterInfo.h" 26 #include "llvm/CodeGen/RegisterScavenging.h" 27 #include "llvm/CodeGen/TargetFrameLowering.h" 28 #include "llvm/IR/DebugInfoMetadata.h" 29 #include "llvm/IR/DiagnosticInfo.h" 30 #include "llvm/IR/Function.h" 31 #include "llvm/Target/TargetOptions.h" 32 #include "llvm/TargetParser/Triple.h" 33 34 using namespace llvm; 35 36 #define GET_CC_REGISTER_LISTS 37 #include "AArch64GenCallingConv.inc" 38 #define GET_REGINFO_TARGET_DESC 39 #include "AArch64GenRegisterInfo.inc" 40 41 AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT) 42 : AArch64GenRegisterInfo(AArch64::LR), TT(TT) { 43 AArch64_MC::initLLVMToCVRegMapping(this); 44 } 45 46 /// Return whether the register needs a CFI entry. Not all unwinders may know 47 /// about SVE registers, so we assume the lowest common denominator, i.e. the 48 /// callee-saves required by the base ABI. For the SVE registers z8-z15 only the 49 /// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is 50 /// returned in \p RegToUseForCFI. 51 bool AArch64RegisterInfo::regNeedsCFI(unsigned Reg, 52 unsigned &RegToUseForCFI) const { 53 if (AArch64::PPRRegClass.contains(Reg)) 54 return false; 55 56 if (AArch64::ZPRRegClass.contains(Reg)) { 57 RegToUseForCFI = getSubReg(Reg, AArch64::dsub); 58 for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) { 59 if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI) 60 return true; 61 } 62 return false; 63 } 64 65 RegToUseForCFI = Reg; 66 return true; 67 } 68 69 const MCPhysReg * 70 AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { 71 assert(MF && "Invalid MachineFunction pointer."); 72 73 if (MF->getFunction().getCallingConv() == CallingConv::GHC) 74 // GHC set of callee saved regs is empty as all those regs are 75 // used for passing STG regs around 76 return CSR_AArch64_NoRegs_SaveList; 77 if (MF->getFunction().getCallingConv() == CallingConv::PreserveNone) 78 return CSR_AArch64_NoneRegs_SaveList; 79 if (MF->getFunction().getCallingConv() == CallingConv::AnyReg) 80 return CSR_AArch64_AllRegs_SaveList; 81 82 if (MF->getFunction().getCallingConv() == CallingConv::ARM64EC_Thunk_X64) 83 return CSR_Win_AArch64_Arm64EC_Thunk_SaveList; 84 85 // Darwin has its own CSR_AArch64_AAPCS_SaveList, which means most CSR save 86 // lists depending on that will need to have their Darwin variant as well. 87 if (MF->getSubtarget<AArch64Subtarget>().isTargetDarwin()) 88 return getDarwinCalleeSavedRegs(MF); 89 90 if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check) 91 return CSR_Win_AArch64_CFGuard_Check_SaveList; 92 if (MF->getSubtarget<AArch64Subtarget>().isTargetWindows()) { 93 if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() 94 ->supportSwiftError() && 95 MF->getFunction().getAttributes().hasAttrSomewhere( 96 Attribute::SwiftError)) 97 return CSR_Win_AArch64_AAPCS_SwiftError_SaveList; 98 if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) 99 return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList; 100 return CSR_Win_AArch64_AAPCS_SaveList; 101 } 102 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) 103 return CSR_AArch64_AAVPCS_SaveList; 104 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall) 105 return CSR_AArch64_SVE_AAPCS_SaveList; 106 if (MF->getFunction().getCallingConv() == 107 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) 108 report_fatal_error( 109 "Calling convention " 110 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only " 111 "supported to improve calls to SME ACLE save/restore/disable-za " 112 "functions, and is not intended to be used beyond that scope."); 113 if (MF->getFunction().getCallingConv() == 114 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) 115 report_fatal_error( 116 "Calling convention " 117 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is " 118 "only supported to improve calls to SME ACLE __arm_get_current_vg " 119 "function, and is not intended to be used beyond that scope."); 120 if (MF->getFunction().getCallingConv() == 121 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) 122 report_fatal_error( 123 "Calling convention " 124 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " 125 "only supported to improve calls to SME ACLE __arm_sme_state " 126 "and is not intended to be used beyond that scope."); 127 if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() 128 ->supportSwiftError() && 129 MF->getFunction().getAttributes().hasAttrSomewhere( 130 Attribute::SwiftError)) 131 return CSR_AArch64_AAPCS_SwiftError_SaveList; 132 if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) 133 return CSR_AArch64_AAPCS_SwiftTail_SaveList; 134 if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) 135 return CSR_AArch64_RT_MostRegs_SaveList; 136 if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll) 137 return CSR_AArch64_RT_AllRegs_SaveList; 138 if (MF->getFunction().getCallingConv() == CallingConv::Win64) 139 // This is for OSes other than Windows; Windows is a separate case further 140 // above. 141 return CSR_AArch64_AAPCS_X18_SaveList; 142 if (MF->getInfo<AArch64FunctionInfo>()->isSVECC()) 143 return CSR_AArch64_SVE_AAPCS_SaveList; 144 return CSR_AArch64_AAPCS_SaveList; 145 } 146 147 const MCPhysReg * 148 AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const { 149 assert(MF && "Invalid MachineFunction pointer."); 150 assert(MF->getSubtarget<AArch64Subtarget>().isTargetDarwin() && 151 "Invalid subtarget for getDarwinCalleeSavedRegs"); 152 153 if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check) 154 report_fatal_error( 155 "Calling convention CFGuard_Check is unsupported on Darwin."); 156 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) 157 return CSR_Darwin_AArch64_AAVPCS_SaveList; 158 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall) 159 report_fatal_error( 160 "Calling convention SVE_VectorCall is unsupported on Darwin."); 161 if (MF->getFunction().getCallingConv() == 162 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) 163 report_fatal_error( 164 "Calling convention " 165 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " 166 "only supported to improve calls to SME ACLE save/restore/disable-za " 167 "functions, and is not intended to be used beyond that scope."); 168 if (MF->getFunction().getCallingConv() == 169 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) 170 report_fatal_error( 171 "Calling convention " 172 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is " 173 "only supported to improve calls to SME ACLE __arm_get_current_vg " 174 "function, and is not intended to be used beyond that scope."); 175 if (MF->getFunction().getCallingConv() == 176 CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) 177 report_fatal_error( 178 "Calling convention " 179 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " 180 "only supported to improve calls to SME ACLE __arm_sme_state " 181 "and is not intended to be used beyond that scope."); 182 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS) 183 return MF->getInfo<AArch64FunctionInfo>()->isSplitCSR() 184 ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList 185 : CSR_Darwin_AArch64_CXX_TLS_SaveList; 186 if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() 187 ->supportSwiftError() && 188 MF->getFunction().getAttributes().hasAttrSomewhere( 189 Attribute::SwiftError)) 190 return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList; 191 if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) 192 return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList; 193 if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) 194 return CSR_Darwin_AArch64_RT_MostRegs_SaveList; 195 if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll) 196 return CSR_Darwin_AArch64_RT_AllRegs_SaveList; 197 if (MF->getFunction().getCallingConv() == CallingConv::Win64) 198 return CSR_Darwin_AArch64_AAPCS_Win64_SaveList; 199 if (MF->getInfo<AArch64FunctionInfo>()->isSVECC()) 200 return CSR_Darwin_AArch64_SVE_AAPCS_SaveList; 201 return CSR_Darwin_AArch64_AAPCS_SaveList; 202 } 203 204 const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy( 205 const MachineFunction *MF) const { 206 assert(MF && "Invalid MachineFunction pointer."); 207 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS && 208 MF->getInfo<AArch64FunctionInfo>()->isSplitCSR()) 209 return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList; 210 return nullptr; 211 } 212 213 void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs( 214 MachineFunction &MF) const { 215 const MCPhysReg *CSRs = getCalleeSavedRegs(&MF); 216 SmallVector<MCPhysReg, 32> UpdatedCSRs; 217 for (const MCPhysReg *I = CSRs; *I; ++I) 218 UpdatedCSRs.push_back(*I); 219 220 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) { 221 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) { 222 UpdatedCSRs.push_back(AArch64::GPR64commonRegClass.getRegister(i)); 223 } 224 } 225 // Register lists are zero-terminated. 226 UpdatedCSRs.push_back(0); 227 MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs); 228 } 229 230 const TargetRegisterClass * 231 AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC, 232 unsigned Idx) const { 233 // edge case for GPR/FPR register classes 234 if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub) 235 return &AArch64::FPR32RegClass; 236 else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub) 237 return &AArch64::FPR64RegClass; 238 239 // Forward to TableGen's default version. 240 return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx); 241 } 242 243 const uint32_t * 244 AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF, 245 CallingConv::ID CC) const { 246 assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() && 247 "Invalid subtarget for getDarwinCallPreservedMask"); 248 249 if (CC == CallingConv::CXX_FAST_TLS) 250 return CSR_Darwin_AArch64_CXX_TLS_RegMask; 251 if (CC == CallingConv::AArch64_VectorCall) 252 return CSR_Darwin_AArch64_AAVPCS_RegMask; 253 if (CC == CallingConv::AArch64_SVE_VectorCall) 254 return CSR_Darwin_AArch64_SVE_AAPCS_RegMask; 255 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) 256 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; 257 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) 258 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask; 259 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) 260 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; 261 if (CC == CallingConv::CFGuard_Check) 262 report_fatal_error( 263 "Calling convention CFGuard_Check is unsupported on Darwin."); 264 if (MF.getSubtarget<AArch64Subtarget>() 265 .getTargetLowering() 266 ->supportSwiftError() && 267 MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError)) 268 return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask; 269 if (CC == CallingConv::SwiftTail) 270 return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask; 271 if (CC == CallingConv::PreserveMost) 272 return CSR_Darwin_AArch64_RT_MostRegs_RegMask; 273 if (CC == CallingConv::PreserveAll) 274 return CSR_Darwin_AArch64_RT_AllRegs_RegMask; 275 return CSR_Darwin_AArch64_AAPCS_RegMask; 276 } 277 278 const uint32_t * 279 AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF, 280 CallingConv::ID CC) const { 281 bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack); 282 if (CC == CallingConv::GHC) 283 // This is academic because all GHC calls are (supposed to be) tail calls 284 return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask; 285 if (CC == CallingConv::PreserveNone) 286 return SCS ? CSR_AArch64_NoneRegs_SCS_RegMask 287 : CSR_AArch64_NoneRegs_RegMask; 288 if (CC == CallingConv::AnyReg) 289 return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask; 290 291 // All the following calling conventions are handled differently on Darwin. 292 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) { 293 if (SCS) 294 report_fatal_error("ShadowCallStack attribute not supported on Darwin."); 295 return getDarwinCallPreservedMask(MF, CC); 296 } 297 298 if (CC == CallingConv::AArch64_VectorCall) 299 return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask; 300 if (CC == CallingConv::AArch64_SVE_VectorCall) 301 return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask 302 : CSR_AArch64_SVE_AAPCS_RegMask; 303 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) 304 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; 305 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) 306 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask; 307 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) 308 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; 309 if (CC == CallingConv::CFGuard_Check) 310 return CSR_Win_AArch64_CFGuard_Check_RegMask; 311 if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering() 312 ->supportSwiftError() && 313 MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError)) 314 return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask 315 : CSR_AArch64_AAPCS_SwiftError_RegMask; 316 if (CC == CallingConv::SwiftTail) { 317 if (SCS) 318 report_fatal_error("ShadowCallStack attribute not supported with swifttail"); 319 return CSR_AArch64_AAPCS_SwiftTail_RegMask; 320 } 321 if (CC == CallingConv::PreserveMost) 322 return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask 323 : CSR_AArch64_RT_MostRegs_RegMask; 324 if (CC == CallingConv::PreserveAll) 325 return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask 326 : CSR_AArch64_RT_AllRegs_RegMask; 327 328 return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask; 329 } 330 331 const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask( 332 const MachineFunction &MF) const { 333 if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux()) 334 return CSR_AArch64_AAPCS_RegMask; 335 336 return nullptr; 337 } 338 339 const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const { 340 if (TT.isOSDarwin()) 341 return CSR_Darwin_AArch64_TLS_RegMask; 342 343 assert(TT.isOSBinFormatELF() && "Invalid target"); 344 return CSR_AArch64_TLS_ELF_RegMask; 345 } 346 347 void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF, 348 const uint32_t **Mask) const { 349 uint32_t *UpdatedMask = MF.allocateRegMask(); 350 unsigned RegMaskSize = MachineOperand::getRegMaskSize(getNumRegs()); 351 memcpy(UpdatedMask, *Mask, sizeof(UpdatedMask[0]) * RegMaskSize); 352 353 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) { 354 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) { 355 for (MCPhysReg SubReg : 356 subregs_inclusive(AArch64::GPR64commonRegClass.getRegister(i))) { 357 // See TargetRegisterInfo::getCallPreservedMask for how to interpret the 358 // register mask. 359 UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32); 360 } 361 } 362 } 363 *Mask = UpdatedMask; 364 } 365 366 const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const { 367 return CSR_AArch64_SMStartStop_RegMask; 368 } 369 370 const uint32_t * 371 AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const { 372 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; 373 } 374 375 const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const { 376 return CSR_AArch64_NoRegs_RegMask; 377 } 378 379 const uint32_t * 380 AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF, 381 CallingConv::ID CC) const { 382 // This should return a register mask that is the same as that returned by 383 // getCallPreservedMask but that additionally preserves the register used for 384 // the first i64 argument (which must also be the register used to return a 385 // single i64 return value) 386 // 387 // In case that the calling convention does not use the same register for 388 // both, the function should return NULL (does not currently apply) 389 assert(CC != CallingConv::GHC && "should not be GHC calling convention."); 390 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) 391 return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask; 392 return CSR_AArch64_AAPCS_ThisReturn_RegMask; 393 } 394 395 const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const { 396 return CSR_AArch64_StackProbe_Windows_RegMask; 397 } 398 399 std::optional<std::string> 400 AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF, 401 MCRegister PhysReg) const { 402 if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19)) 403 return std::string("X19 is used as the frame base pointer register."); 404 405 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) { 406 bool warn = false; 407 if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) || 408 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X14) || 409 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X23) || 410 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X24) || 411 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28)) 412 warn = true; 413 414 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i) 415 if (MCRegisterInfo::regsOverlap(PhysReg, i)) 416 warn = true; 417 418 if (warn) 419 return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) + 420 " is clobbered by asynchronous signals when using Arm64EC."; 421 } 422 423 return {}; 424 } 425 426 BitVector 427 AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const { 428 const AArch64FrameLowering *TFI = getFrameLowering(MF); 429 430 // FIXME: avoid re-calculating this every time. 431 BitVector Reserved(getNumRegs()); 432 markSuperRegs(Reserved, AArch64::WSP); 433 markSuperRegs(Reserved, AArch64::WZR); 434 435 if (TFI->hasFP(MF) || TT.isOSDarwin()) 436 markSuperRegs(Reserved, AArch64::W29); 437 438 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) { 439 // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous 440 // signals, so we can't ever use them. 441 markSuperRegs(Reserved, AArch64::W13); 442 markSuperRegs(Reserved, AArch64::W14); 443 markSuperRegs(Reserved, AArch64::W23); 444 markSuperRegs(Reserved, AArch64::W24); 445 markSuperRegs(Reserved, AArch64::W28); 446 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i) 447 markSuperRegs(Reserved, i); 448 } 449 450 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { 451 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i)) 452 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); 453 } 454 455 if (hasBasePointer(MF)) 456 markSuperRegs(Reserved, AArch64::W19); 457 458 // SLH uses register W16/X16 as the taint register. 459 if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) 460 markSuperRegs(Reserved, AArch64::W16); 461 462 // FFR is modelled as global state that cannot be allocated. 463 if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) 464 Reserved.set(AArch64::FFR); 465 466 // SME tiles are not allocatable. 467 if (MF.getSubtarget<AArch64Subtarget>().hasSME()) { 468 for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA)) 469 Reserved.set(SubReg); 470 } 471 472 // VG cannot be allocated 473 Reserved.set(AArch64::VG); 474 475 if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) { 476 for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true); 477 SubReg.isValid(); ++SubReg) 478 Reserved.set(*SubReg); 479 } 480 481 markSuperRegs(Reserved, AArch64::FPCR); 482 markSuperRegs(Reserved, AArch64::FPMR); 483 markSuperRegs(Reserved, AArch64::FPSR); 484 485 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) { 486 markSuperRegs(Reserved, AArch64::X27); 487 markSuperRegs(Reserved, AArch64::X28); 488 markSuperRegs(Reserved, AArch64::W27); 489 markSuperRegs(Reserved, AArch64::W28); 490 } 491 492 assert(checkAllSuperRegsMarked(Reserved)); 493 494 // Add _HI registers after checkAllSuperRegsMarked as this check otherwise 495 // becomes considerably more expensive. 496 Reserved.set(AArch64::WSP_HI); 497 Reserved.set(AArch64::WZR_HI); 498 static_assert(AArch64::W30_HI - AArch64::W0_HI == 30, 499 "Unexpected order of registers"); 500 Reserved.set(AArch64::W0_HI, AArch64::W30_HI); 501 static_assert(AArch64::B31_HI - AArch64::B0_HI == 31, 502 "Unexpected order of registers"); 503 Reserved.set(AArch64::B0_HI, AArch64::B31_HI); 504 static_assert(AArch64::H31_HI - AArch64::H0_HI == 31, 505 "Unexpected order of registers"); 506 Reserved.set(AArch64::H0_HI, AArch64::H31_HI); 507 static_assert(AArch64::S31_HI - AArch64::S0_HI == 31, 508 "Unexpected order of registers"); 509 Reserved.set(AArch64::S0_HI, AArch64::S31_HI); 510 static_assert(AArch64::D31_HI - AArch64::D0_HI == 31, 511 "Unexpected order of registers"); 512 Reserved.set(AArch64::D0_HI, AArch64::D31_HI); 513 static_assert(AArch64::Q31_HI - AArch64::Q0_HI == 31, 514 "Unexpected order of registers"); 515 Reserved.set(AArch64::Q0_HI, AArch64::Q31_HI); 516 517 return Reserved; 518 } 519 520 BitVector 521 AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const { 522 BitVector Reserved(getNumRegs()); 523 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { 524 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i)) 525 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); 526 } 527 528 if (MF.getSubtarget<AArch64Subtarget>().isLRReservedForRA()) { 529 // In order to prevent the register allocator from using LR, we need to 530 // mark it as reserved. However we don't want to keep it reserved throughout 531 // the pipeline since it prevents other infrastructure from reasoning about 532 // it's liveness. We use the NoVRegs property instead of IsSSA because 533 // IsSSA is removed before VirtRegRewriter runs. 534 if (!MF.getProperties().hasProperty( 535 MachineFunctionProperties::Property::NoVRegs)) 536 markSuperRegs(Reserved, AArch64::LR); 537 } 538 539 assert(checkAllSuperRegsMarked(Reserved)); 540 541 // Handle strictlyReservedRegs separately to avoid re-evaluating the assert, 542 // which becomes considerably expensive when considering the _HI registers. 543 Reserved |= getStrictlyReservedRegs(MF); 544 545 return Reserved; 546 } 547 548 bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF, 549 MCRegister Reg) const { 550 return getReservedRegs(MF)[Reg]; 551 } 552 553 bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF, 554 MCRegister Reg) const { 555 return getStrictlyReservedRegs(MF)[Reg]; 556 } 557 558 bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const { 559 return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) { 560 return isStrictlyReservedReg(MF, r); 561 }); 562 } 563 564 void AArch64RegisterInfo::emitReservedArgRegCallError( 565 const MachineFunction &MF) const { 566 const Function &F = MF.getFunction(); 567 F.getContext().diagnose(DiagnosticInfoUnsupported{F, ("AArch64 doesn't support" 568 " function calls if any of the argument registers is reserved.")}); 569 } 570 571 bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF, 572 MCRegister PhysReg) const { 573 // SLH uses register X16 as the taint register but it will fallback to a different 574 // method if the user clobbers it. So X16 is not reserved for inline asm but is 575 // for normal codegen. 576 if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening) && 577 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X16)) 578 return true; 579 580 // ZA/ZT0 registers are reserved but may be permitted in the clobber list. 581 if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0) 582 return true; 583 584 return !isReservedReg(MF, PhysReg); 585 } 586 587 const TargetRegisterClass * 588 AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF, 589 unsigned Kind) const { 590 return &AArch64::GPR64spRegClass; 591 } 592 593 const TargetRegisterClass * 594 AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { 595 if (RC == &AArch64::CCRRegClass) 596 return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV. 597 return RC; 598 } 599 600 unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; } 601 602 bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { 603 const MachineFrameInfo &MFI = MF.getFrameInfo(); 604 605 // In the presence of variable sized objects or funclets, if the fixed stack 606 // size is large enough that referencing from the FP won't result in things 607 // being in range relatively often, we can use a base pointer to allow access 608 // from the other direction like the SP normally works. 609 // 610 // Furthermore, if both variable sized objects are present, and the 611 // stack needs to be dynamically re-aligned, the base pointer is the only 612 // reliable way to reference the locals. 613 if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) { 614 if (hasStackRealignment(MF)) 615 return true; 616 617 auto &ST = MF.getSubtarget<AArch64Subtarget>(); 618 if (ST.hasSVE() || ST.isStreaming()) { 619 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); 620 // Frames that have variable sized objects and scalable SVE objects, 621 // should always use a basepointer. 622 if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE()) 623 return true; 624 } 625 626 // Conservatively estimate whether the negative offset from the frame 627 // pointer will be sufficient to reach. If a function has a smallish 628 // frame, it's less likely to have lots of spills and callee saved 629 // space, so it's all more likely to be within range of the frame pointer. 630 // If it's wrong, we'll materialize the constant and still get to the 631 // object; it's just suboptimal. Negative offsets use the unscaled 632 // load/store instructions, which have a 9-bit signed immediate. 633 return MFI.getLocalFrameSize() >= 256; 634 } 635 636 return false; 637 } 638 639 bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF, 640 MCRegister Reg) const { 641 CallingConv::ID CC = MF.getFunction().getCallingConv(); 642 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); 643 bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv(), 644 MF.getFunction().isVarArg()); 645 646 auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) { 647 return llvm::is_contained(RegList, Reg); 648 }; 649 650 switch (CC) { 651 default: 652 report_fatal_error("Unsupported calling convention."); 653 case CallingConv::GHC: 654 return HasReg(CC_AArch64_GHC_ArgRegs, Reg); 655 case CallingConv::PreserveNone: 656 if (!MF.getFunction().isVarArg()) 657 return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg); 658 [[fallthrough]]; 659 case CallingConv::C: 660 case CallingConv::Fast: 661 case CallingConv::PreserveMost: 662 case CallingConv::PreserveAll: 663 case CallingConv::CXX_FAST_TLS: 664 case CallingConv::Swift: 665 case CallingConv::SwiftTail: 666 case CallingConv::Tail: 667 if (STI.isTargetWindows()) { 668 if (IsVarArg) 669 return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg); 670 switch (CC) { 671 default: 672 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); 673 case CallingConv::Swift: 674 case CallingConv::SwiftTail: 675 return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) || 676 HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); 677 } 678 } 679 if (!STI.isTargetDarwin()) { 680 switch (CC) { 681 default: 682 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg); 683 case CallingConv::Swift: 684 case CallingConv::SwiftTail: 685 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) || 686 HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg); 687 } 688 } 689 if (!IsVarArg) { 690 switch (CC) { 691 default: 692 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg); 693 case CallingConv::Swift: 694 case CallingConv::SwiftTail: 695 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) || 696 HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg); 697 } 698 } 699 if (STI.isTargetILP32()) 700 return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg); 701 return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg); 702 case CallingConv::Win64: 703 if (IsVarArg) 704 HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg); 705 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); 706 case CallingConv::CFGuard_Check: 707 return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg); 708 case CallingConv::AArch64_VectorCall: 709 case CallingConv::AArch64_SVE_VectorCall: 710 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0: 711 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1: 712 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2: 713 if (STI.isTargetWindows()) 714 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); 715 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg); 716 } 717 } 718 719 Register 720 AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const { 721 const AArch64FrameLowering *TFI = getFrameLowering(MF); 722 return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP; 723 } 724 725 bool AArch64RegisterInfo::requiresRegisterScavenging( 726 const MachineFunction &MF) const { 727 return true; 728 } 729 730 bool AArch64RegisterInfo::requiresVirtualBaseRegisters( 731 const MachineFunction &MF) const { 732 return true; 733 } 734 735 bool 736 AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const { 737 // This function indicates whether the emergency spillslot should be placed 738 // close to the beginning of the stackframe (closer to FP) or the end 739 // (closer to SP). 740 // 741 // The beginning works most reliably if we have a frame pointer. 742 // In the presence of any non-constant space between FP and locals, 743 // (e.g. in case of stack realignment or a scalable SVE area), it is 744 // better to use SP or BP. 745 const AArch64FrameLowering &TFI = *getFrameLowering(MF); 746 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); 747 assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() || 748 AFI->hasCalculatedStackSizeSVE()) && 749 "Expected SVE area to be calculated by this point"); 750 return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE(); 751 } 752 753 bool AArch64RegisterInfo::requiresFrameIndexScavenging( 754 const MachineFunction &MF) const { 755 return true; 756 } 757 758 bool 759 AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const { 760 const MachineFrameInfo &MFI = MF.getFrameInfo(); 761 if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack()) 762 return true; 763 return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken(); 764 } 765 766 /// needsFrameBaseReg - Returns true if the instruction's frame index 767 /// reference would be better served by a base register other than FP 768 /// or SP. Used by LocalStackFrameAllocation to determine which frame index 769 /// references it should create new base registers for. 770 bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI, 771 int64_t Offset) const { 772 for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i) 773 assert(i < MI->getNumOperands() && 774 "Instr doesn't have FrameIndex operand!"); 775 776 // It's the load/store FI references that cause issues, as it can be difficult 777 // to materialize the offset if it won't fit in the literal field. Estimate 778 // based on the size of the local frame and some conservative assumptions 779 // about the rest of the stack frame (note, this is pre-regalloc, so 780 // we don't know everything for certain yet) whether this offset is likely 781 // to be out of range of the immediate. Return true if so. 782 783 // We only generate virtual base registers for loads and stores, so 784 // return false for everything else. 785 if (!MI->mayLoad() && !MI->mayStore()) 786 return false; 787 788 // Without a virtual base register, if the function has variable sized 789 // objects, all fixed-size local references will be via the frame pointer, 790 // Approximate the offset and see if it's legal for the instruction. 791 // Note that the incoming offset is based on the SP value at function entry, 792 // so it'll be negative. 793 MachineFunction &MF = *MI->getParent()->getParent(); 794 const AArch64FrameLowering *TFI = getFrameLowering(MF); 795 MachineFrameInfo &MFI = MF.getFrameInfo(); 796 797 // Estimate an offset from the frame pointer. 798 // Conservatively assume all GPR callee-saved registers get pushed. 799 // FP, LR, X19-X28, D8-D15. 64-bits each. 800 int64_t FPOffset = Offset - 16 * 20; 801 // Estimate an offset from the stack pointer. 802 // The incoming offset is relating to the SP at the start of the function, 803 // but when we access the local it'll be relative to the SP after local 804 // allocation, so adjust our SP-relative offset by that allocation size. 805 Offset += MFI.getLocalFrameSize(); 806 // Assume that we'll have at least some spill slots allocated. 807 // FIXME: This is a total SWAG number. We should run some statistics 808 // and pick a real one. 809 Offset += 128; // 128 bytes of spill slots 810 811 // If there is a frame pointer, try using it. 812 // The FP is only available if there is no dynamic realignment. We 813 // don't know for sure yet whether we'll need that, so we guess based 814 // on whether there are any local variables that would trigger it. 815 if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset)) 816 return false; 817 818 // If we can reference via the stack pointer or base pointer, try that. 819 // FIXME: This (and the code that resolves the references) can be improved 820 // to only disallow SP relative references in the live range of 821 // the VLA(s). In practice, it's unclear how much difference that 822 // would make, but it may be worth doing. 823 if (isFrameOffsetLegal(MI, AArch64::SP, Offset)) 824 return false; 825 826 // If even offset 0 is illegal, we don't want a virtual base register. 827 if (!isFrameOffsetLegal(MI, AArch64::SP, 0)) 828 return false; 829 830 // The offset likely isn't legal; we want to allocate a virtual base register. 831 return true; 832 } 833 834 bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, 835 Register BaseReg, 836 int64_t Offset) const { 837 assert(MI && "Unable to get the legal offset for nil instruction."); 838 StackOffset SaveOffset = StackOffset::getFixed(Offset); 839 return isAArch64FrameOffsetLegal(*MI, SaveOffset) & AArch64FrameOffsetIsLegal; 840 } 841 842 /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx 843 /// at the beginning of the basic block. 844 Register 845 AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, 846 int FrameIdx, 847 int64_t Offset) const { 848 MachineBasicBlock::iterator Ins = MBB->begin(); 849 DebugLoc DL; // Defaults to "unknown" 850 if (Ins != MBB->end()) 851 DL = Ins->getDebugLoc(); 852 const MachineFunction &MF = *MBB->getParent(); 853 const AArch64InstrInfo *TII = 854 MF.getSubtarget<AArch64Subtarget>().getInstrInfo(); 855 const MCInstrDesc &MCID = TII->get(AArch64::ADDXri); 856 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 857 Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); 858 MRI.constrainRegClass(BaseReg, TII->getRegClass(MCID, 0, this, MF)); 859 unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0); 860 861 BuildMI(*MBB, Ins, DL, MCID, BaseReg) 862 .addFrameIndex(FrameIdx) 863 .addImm(Offset) 864 .addImm(Shifter); 865 866 return BaseReg; 867 } 868 869 void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, 870 int64_t Offset) const { 871 // ARM doesn't need the general 64-bit offsets 872 StackOffset Off = StackOffset::getFixed(Offset); 873 874 unsigned i = 0; 875 while (!MI.getOperand(i).isFI()) { 876 ++i; 877 assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!"); 878 } 879 880 const MachineFunction *MF = MI.getParent()->getParent(); 881 const AArch64InstrInfo *TII = 882 MF->getSubtarget<AArch64Subtarget>().getInstrInfo(); 883 bool Done = rewriteAArch64FrameIndex(MI, i, BaseReg, Off, TII); 884 assert(Done && "Unable to resolve frame index!"); 885 (void)Done; 886 } 887 888 // Create a scratch register for the frame index elimination in an instruction. 889 // This function has special handling of stack tagging loop pseudos, in which 890 // case it can also change the instruction opcode. 891 static Register 892 createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum, 893 const AArch64InstrInfo *TII) { 894 // ST*Gloop have a reserved scratch register in operand 1. Use it, and also 895 // replace the instruction with the writeback variant because it will now 896 // satisfy the operand constraints for it. 897 Register ScratchReg; 898 if (MI.getOpcode() == AArch64::STGloop || 899 MI.getOpcode() == AArch64::STZGloop) { 900 assert(FIOperandNum == 3 && 901 "Wrong frame index operand for STGloop/STZGloop"); 902 unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback 903 : AArch64::STZGloop_wback; 904 ScratchReg = MI.getOperand(1).getReg(); 905 MI.getOperand(3).ChangeToRegister(ScratchReg, false, false, true); 906 MI.setDesc(TII->get(Op)); 907 MI.tieOperands(1, 3); 908 } else { 909 ScratchReg = 910 MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); 911 MI.getOperand(FIOperandNum) 912 .ChangeToRegister(ScratchReg, false, false, true); 913 } 914 return ScratchReg; 915 } 916 917 void AArch64RegisterInfo::getOffsetOpcodes( 918 const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const { 919 // The smallest scalable element supported by scaled SVE addressing 920 // modes are predicates, which are 2 scalable bytes in size. So the scalable 921 // byte offset must always be a multiple of 2. 922 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset"); 923 924 // Add fixed-sized offset using existing DIExpression interface. 925 DIExpression::appendOffset(Ops, Offset.getFixed()); 926 927 unsigned VG = getDwarfRegNum(AArch64::VG, true); 928 int64_t VGSized = Offset.getScalable() / 2; 929 if (VGSized > 0) { 930 Ops.push_back(dwarf::DW_OP_constu); 931 Ops.push_back(VGSized); 932 Ops.append({dwarf::DW_OP_bregx, VG, 0ULL}); 933 Ops.push_back(dwarf::DW_OP_mul); 934 Ops.push_back(dwarf::DW_OP_plus); 935 } else if (VGSized < 0) { 936 Ops.push_back(dwarf::DW_OP_constu); 937 Ops.push_back(-VGSized); 938 Ops.append({dwarf::DW_OP_bregx, VG, 0ULL}); 939 Ops.push_back(dwarf::DW_OP_mul); 940 Ops.push_back(dwarf::DW_OP_minus); 941 } 942 } 943 944 bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, 945 int SPAdj, unsigned FIOperandNum, 946 RegScavenger *RS) const { 947 assert(SPAdj == 0 && "Unexpected"); 948 949 MachineInstr &MI = *II; 950 MachineBasicBlock &MBB = *MI.getParent(); 951 MachineFunction &MF = *MBB.getParent(); 952 const MachineFrameInfo &MFI = MF.getFrameInfo(); 953 const AArch64InstrInfo *TII = 954 MF.getSubtarget<AArch64Subtarget>().getInstrInfo(); 955 const AArch64FrameLowering *TFI = getFrameLowering(MF); 956 int FrameIndex = MI.getOperand(FIOperandNum).getIndex(); 957 bool Tagged = 958 MI.getOperand(FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED; 959 Register FrameReg; 960 961 // Special handling of dbg_value, stackmap patchpoint statepoint instructions. 962 if (MI.getOpcode() == TargetOpcode::STACKMAP || 963 MI.getOpcode() == TargetOpcode::PATCHPOINT || 964 MI.getOpcode() == TargetOpcode::STATEPOINT) { 965 StackOffset Offset = 966 TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg, 967 /*PreferFP=*/true, 968 /*ForSimm=*/false); 969 Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm()); 970 MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/); 971 MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset.getFixed()); 972 return false; 973 } 974 975 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) { 976 MachineOperand &FI = MI.getOperand(FIOperandNum); 977 StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex); 978 assert(!Offset.getScalable() && 979 "Frame offsets with a scalable component are not supported"); 980 FI.ChangeToImmediate(Offset.getFixed()); 981 return false; 982 } 983 984 StackOffset Offset; 985 if (MI.getOpcode() == AArch64::TAGPstack) { 986 // TAGPstack must use the virtual frame register in its 3rd operand. 987 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); 988 FrameReg = MI.getOperand(3).getReg(); 989 Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) + 990 AFI->getTaggedBasePointerOffset()); 991 } else if (Tagged) { 992 StackOffset SPOffset = StackOffset::getFixed( 993 MFI.getObjectOffset(FrameIndex) + (int64_t)MFI.getStackSize()); 994 if (MFI.hasVarSizedObjects() || 995 isAArch64FrameOffsetLegal(MI, SPOffset, nullptr, nullptr, nullptr) != 996 (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) { 997 // Can't update to SP + offset in place. Precalculate the tagged pointer 998 // in a scratch register. 999 Offset = TFI->resolveFrameIndexReference( 1000 MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); 1001 Register ScratchReg = 1002 MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); 1003 emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, 1004 TII); 1005 BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg) 1006 .addReg(ScratchReg) 1007 .addReg(ScratchReg) 1008 .addImm(0); 1009 MI.getOperand(FIOperandNum) 1010 .ChangeToRegister(ScratchReg, false, false, true); 1011 return false; 1012 } 1013 FrameReg = AArch64::SP; 1014 Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) + 1015 (int64_t)MFI.getStackSize()); 1016 } else { 1017 Offset = TFI->resolveFrameIndexReference( 1018 MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); 1019 } 1020 1021 // Modify MI as necessary to handle as much of 'Offset' as possible 1022 if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII)) 1023 return true; 1024 1025 assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) && 1026 "Emergency spill slot is out of reach"); 1027 1028 // If we get here, the immediate doesn't fit into the instruction. We folded 1029 // as much as possible above. Handle the rest, providing a register that is 1030 // SP+LargeImm. 1031 Register ScratchReg = 1032 createScratchRegisterForInstruction(MI, FIOperandNum, TII); 1033 emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII); 1034 return false; 1035 } 1036 1037 unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, 1038 MachineFunction &MF) const { 1039 const AArch64FrameLowering *TFI = getFrameLowering(MF); 1040 1041 switch (RC->getID()) { 1042 default: 1043 return 0; 1044 case AArch64::GPR32RegClassID: 1045 case AArch64::GPR32spRegClassID: 1046 case AArch64::GPR32allRegClassID: 1047 case AArch64::GPR64spRegClassID: 1048 case AArch64::GPR64allRegClassID: 1049 case AArch64::GPR64RegClassID: 1050 case AArch64::GPR32commonRegClassID: 1051 case AArch64::GPR64commonRegClassID: 1052 return 32 - 1 // XZR/SP 1053 - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP 1054 - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved() 1055 - hasBasePointer(MF); // X19 1056 case AArch64::FPR8RegClassID: 1057 case AArch64::FPR16RegClassID: 1058 case AArch64::FPR32RegClassID: 1059 case AArch64::FPR64RegClassID: 1060 case AArch64::FPR128RegClassID: 1061 return 32; 1062 1063 case AArch64::MatrixIndexGPR32_8_11RegClassID: 1064 case AArch64::MatrixIndexGPR32_12_15RegClassID: 1065 return 4; 1066 1067 case AArch64::DDRegClassID: 1068 case AArch64::DDDRegClassID: 1069 case AArch64::DDDDRegClassID: 1070 case AArch64::QQRegClassID: 1071 case AArch64::QQQRegClassID: 1072 case AArch64::QQQQRegClassID: 1073 return 32; 1074 1075 case AArch64::FPR128_loRegClassID: 1076 case AArch64::FPR64_loRegClassID: 1077 case AArch64::FPR16_loRegClassID: 1078 return 16; 1079 case AArch64::FPR128_0to7RegClassID: 1080 return 8; 1081 } 1082 } 1083 1084 // FORM_TRANSPOSED_REG_TUPLE nodes are created to improve register allocation 1085 // where a consecutive multi-vector tuple is constructed from the same indices 1086 // of multiple strided loads. This may still result in unnecessary copies 1087 // between the loads and the tuple. Here we try to return a hint to assign the 1088 // contiguous ZPRMulReg starting at the same register as the first operand of 1089 // the pseudo, which should be a subregister of the first strided load. 1090 // 1091 // For example, if the first strided load has been assigned $z16_z20_z24_z28 1092 // and the operands of the pseudo are each accessing subregister zsub2, we 1093 // should look through through Order to find a contiguous register which 1094 // begins with $z24 (i.e. $z24_z25_z26_z27). 1095 // 1096 bool AArch64RegisterInfo::getRegAllocationHints( 1097 Register VirtReg, ArrayRef<MCPhysReg> Order, 1098 SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF, 1099 const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const { 1100 const MachineRegisterInfo &MRI = MF.getRegInfo(); 1101 1102 // The SVE calling convention preserves registers Z8-Z23. As a result, there 1103 // are no ZPR2Strided or ZPR4Strided registers that do not overlap with the 1104 // callee-saved registers and so by default these will be pushed to the back 1105 // of the allocation order for the ZPRStridedOrContiguous classes. 1106 // If any of the instructions which define VirtReg are used by the 1107 // FORM_TRANSPOSED_REG_TUPLE pseudo, we want to favour reducing copy 1108 // instructions over reducing the number of clobbered callee-save registers, 1109 // so we add the strided registers as a hint. 1110 unsigned RegID = MRI.getRegClass(VirtReg)->getID(); 1111 // Look through uses of the register for FORM_TRANSPOSED_REG_TUPLE. 1112 if ((RegID == AArch64::ZPR2StridedOrContiguousRegClassID || 1113 RegID == AArch64::ZPR4StridedOrContiguousRegClassID) && 1114 any_of(MRI.use_nodbg_instructions(VirtReg), [](const MachineInstr &Use) { 1115 return Use.getOpcode() == 1116 AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO || 1117 Use.getOpcode() == AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO; 1118 })) { 1119 const TargetRegisterClass *StridedRC = 1120 RegID == AArch64::ZPR2StridedOrContiguousRegClassID 1121 ? &AArch64::ZPR2StridedRegClass 1122 : &AArch64::ZPR4StridedRegClass; 1123 1124 for (MCPhysReg Reg : Order) 1125 if (StridedRC->contains(Reg)) 1126 Hints.push_back(Reg); 1127 1128 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, 1129 VRM); 1130 } 1131 1132 for (MachineInstr &MI : MRI.def_instructions(VirtReg)) { 1133 if (MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO && 1134 MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO) 1135 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, 1136 MF, VRM); 1137 1138 unsigned FirstOpSubReg = MI.getOperand(1).getSubReg(); 1139 switch (FirstOpSubReg) { 1140 case AArch64::zsub0: 1141 case AArch64::zsub1: 1142 case AArch64::zsub2: 1143 case AArch64::zsub3: 1144 break; 1145 default: 1146 continue; 1147 } 1148 1149 // Look up the physical register mapped to the first operand of the pseudo. 1150 Register FirstOpVirtReg = MI.getOperand(1).getReg(); 1151 if (!VRM->hasPhys(FirstOpVirtReg)) 1152 continue; 1153 1154 MCRegister TupleStartReg = 1155 getSubReg(VRM->getPhys(FirstOpVirtReg), FirstOpSubReg); 1156 for (unsigned I = 0; I < Order.size(); ++I) 1157 if (MCRegister R = getSubReg(Order[I], AArch64::zsub0)) 1158 if (R == TupleStartReg) 1159 Hints.push_back(Order[I]); 1160 } 1161 1162 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, 1163 VRM); 1164 } 1165 1166 unsigned AArch64RegisterInfo::getLocalAddressRegister( 1167 const MachineFunction &MF) const { 1168 const auto &MFI = MF.getFrameInfo(); 1169 if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects()) 1170 return AArch64::SP; 1171 else if (hasStackRealignment(MF)) 1172 return getBaseRegister(); 1173 return getFrameRegister(MF); 1174 } 1175 1176 /// SrcRC and DstRC will be morphed into NewRC if this returns true 1177 bool AArch64RegisterInfo::shouldCoalesce( 1178 MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg, 1179 const TargetRegisterClass *DstRC, unsigned DstSubReg, 1180 const TargetRegisterClass *NewRC, LiveIntervals &LIS) const { 1181 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo(); 1182 1183 if (MI->isCopy() && 1184 ((DstRC->getID() == AArch64::GPR64RegClassID) || 1185 (DstRC->getID() == AArch64::GPR64commonRegClassID)) && 1186 MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg()) 1187 // Do not coalesce in the case of a 32-bit subregister copy 1188 // which implements a 32 to 64 bit zero extension 1189 // which relies on the upper 32 bits being zeroed. 1190 return false; 1191 1192 auto IsCoalescerBarrier = [](const MachineInstr &MI) { 1193 switch (MI.getOpcode()) { 1194 case AArch64::COALESCER_BARRIER_FPR16: 1195 case AArch64::COALESCER_BARRIER_FPR32: 1196 case AArch64::COALESCER_BARRIER_FPR64: 1197 case AArch64::COALESCER_BARRIER_FPR128: 1198 return true; 1199 default: 1200 return false; 1201 } 1202 }; 1203 1204 // For calls that temporarily have to toggle streaming mode as part of the 1205 // call-sequence, we need to be more careful when coalescing copy instructions 1206 // so that we don't end up coalescing the NEON/FP result or argument register 1207 // with a whole Z-register, such that after coalescing the register allocator 1208 // will try to spill/reload the entire Z register. 1209 // 1210 // We do this by checking if the node has any defs/uses that are 1211 // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to 1212 // instruct the coalescer to avoid coalescing the copy. 1213 if (MI->isCopy() && SubReg != DstSubReg && 1214 (AArch64::ZPRRegClass.hasSubClassEq(DstRC) || 1215 AArch64::ZPRRegClass.hasSubClassEq(SrcRC))) { 1216 unsigned SrcReg = MI->getOperand(1).getReg(); 1217 if (any_of(MRI.def_instructions(SrcReg), IsCoalescerBarrier)) 1218 return false; 1219 unsigned DstReg = MI->getOperand(0).getReg(); 1220 if (any_of(MRI.use_nodbg_instructions(DstReg), IsCoalescerBarrier)) 1221 return false; 1222 } 1223 1224 return true; 1225 } 1226 1227 bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo( 1228 MCRegister R) const { 1229 return R == AArch64::VG; 1230 } 1231