1 //===-- RISCVFrameLowering.cpp - RISCV Frame Information ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the RISCV implementation of TargetFrameLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "RISCVFrameLowering.h"
14 #include "RISCVMachineFunctionInfo.h"
15 #include "RISCVSubtarget.h"
16 #include "llvm/BinaryFormat/Dwarf.h"
17 #include "llvm/CodeGen/MachineFrameInfo.h"
18 #include "llvm/CodeGen/MachineFunction.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/RegisterScavenging.h"
22 #include "llvm/IR/DiagnosticInfo.h"
23 #include "llvm/MC/MCDwarf.h"
24 #include "llvm/Support/LEB128.h"
25
26 #include <algorithm>
27
28 using namespace llvm;
29
30 // For now we use x18, a.k.a s2, as pointer to shadow call stack.
31 // User should explicitly set -ffixed-x18 and not use x18 in their asm.
emitSCSPrologue(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & DL)32 static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
33 MachineBasicBlock::iterator MI,
34 const DebugLoc &DL) {
35 if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))
36 return;
37
38 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
39 Register RAReg = STI.getRegisterInfo()->getRARegister();
40
41 // Do not save RA to the SCS if it's not saved to the regular stack,
42 // i.e. RA is not at risk of being overwritten.
43 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
44 if (llvm::none_of(
45 CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
46 return;
47
48 Register SCSPReg = RISCVABI::getSCSPReg();
49
50 auto &Ctx = MF.getFunction().getContext();
51 if (!STI.isRegisterReservedByUser(SCSPReg)) {
52 Ctx.diagnose(DiagnosticInfoUnsupported{
53 MF.getFunction(), "x18 not reserved by user for Shadow Call Stack."});
54 return;
55 }
56
57 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
58 if (RVFI->useSaveRestoreLibCalls(MF)) {
59 Ctx.diagnose(DiagnosticInfoUnsupported{
60 MF.getFunction(),
61 "Shadow Call Stack cannot be combined with Save/Restore LibCalls."});
62 return;
63 }
64
65 const RISCVInstrInfo *TII = STI.getInstrInfo();
66 bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
67 int64_t SlotSize = STI.getXLen() / 8;
68 // Store return address to shadow call stack
69 // s[w|d] ra, 0(s2)
70 // addi s2, s2, [4|8]
71 BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
72 .addReg(RAReg)
73 .addReg(SCSPReg)
74 .addImm(0)
75 .setMIFlag(MachineInstr::FrameSetup);
76 BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
77 .addReg(SCSPReg, RegState::Define)
78 .addReg(SCSPReg)
79 .addImm(SlotSize)
80 .setMIFlag(MachineInstr::FrameSetup);
81 }
82
emitSCSEpilogue(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & DL)83 static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB,
84 MachineBasicBlock::iterator MI,
85 const DebugLoc &DL) {
86 if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack))
87 return;
88
89 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
90 Register RAReg = STI.getRegisterInfo()->getRARegister();
91
92 // See emitSCSPrologue() above.
93 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
94 if (llvm::none_of(
95 CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
96 return;
97
98 Register SCSPReg = RISCVABI::getSCSPReg();
99
100 auto &Ctx = MF.getFunction().getContext();
101 if (!STI.isRegisterReservedByUser(SCSPReg)) {
102 Ctx.diagnose(DiagnosticInfoUnsupported{
103 MF.getFunction(), "x18 not reserved by user for Shadow Call Stack."});
104 return;
105 }
106
107 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
108 if (RVFI->useSaveRestoreLibCalls(MF)) {
109 Ctx.diagnose(DiagnosticInfoUnsupported{
110 MF.getFunction(),
111 "Shadow Call Stack cannot be combined with Save/Restore LibCalls."});
112 return;
113 }
114
115 const RISCVInstrInfo *TII = STI.getInstrInfo();
116 bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit);
117 int64_t SlotSize = STI.getXLen() / 8;
118 // Load return address from shadow call stack
119 // l[w|d] ra, -[4|8](s2)
120 // addi s2, s2, -[4|8]
121 BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW))
122 .addReg(RAReg, RegState::Define)
123 .addReg(SCSPReg)
124 .addImm(-SlotSize)
125 .setMIFlag(MachineInstr::FrameDestroy);
126 BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
127 .addReg(SCSPReg, RegState::Define)
128 .addReg(SCSPReg)
129 .addImm(-SlotSize)
130 .setMIFlag(MachineInstr::FrameDestroy);
131 }
132
133 // Get the ID of the libcall used for spilling and restoring callee saved
134 // registers. The ID is representative of the number of registers saved or
135 // restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
136 // single register.
getLibCallID(const MachineFunction & MF,const std::vector<CalleeSavedInfo> & CSI)137 static int getLibCallID(const MachineFunction &MF,
138 const std::vector<CalleeSavedInfo> &CSI) {
139 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
140
141 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
142 return -1;
143
144 Register MaxReg = RISCV::NoRegister;
145 for (auto &CS : CSI)
146 // RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to
147 // registers which can be saved by libcall.
148 if (CS.getFrameIdx() < 0)
149 MaxReg = std::max(MaxReg.id(), CS.getReg().id());
150
151 if (MaxReg == RISCV::NoRegister)
152 return -1;
153
154 switch (MaxReg) {
155 default:
156 llvm_unreachable("Something has gone wrong!");
157 case /*s11*/ RISCV::X27: return 12;
158 case /*s10*/ RISCV::X26: return 11;
159 case /*s9*/ RISCV::X25: return 10;
160 case /*s8*/ RISCV::X24: return 9;
161 case /*s7*/ RISCV::X23: return 8;
162 case /*s6*/ RISCV::X22: return 7;
163 case /*s5*/ RISCV::X21: return 6;
164 case /*s4*/ RISCV::X20: return 5;
165 case /*s3*/ RISCV::X19: return 4;
166 case /*s2*/ RISCV::X18: return 3;
167 case /*s1*/ RISCV::X9: return 2;
168 case /*s0*/ RISCV::X8: return 1;
169 case /*ra*/ RISCV::X1: return 0;
170 }
171 }
172
173 // Get the name of the libcall used for spilling callee saved registers.
174 // If this function will not use save/restore libcalls, then return a nullptr.
175 static const char *
getSpillLibCallName(const MachineFunction & MF,const std::vector<CalleeSavedInfo> & CSI)176 getSpillLibCallName(const MachineFunction &MF,
177 const std::vector<CalleeSavedInfo> &CSI) {
178 static const char *const SpillLibCalls[] = {
179 "__riscv_save_0",
180 "__riscv_save_1",
181 "__riscv_save_2",
182 "__riscv_save_3",
183 "__riscv_save_4",
184 "__riscv_save_5",
185 "__riscv_save_6",
186 "__riscv_save_7",
187 "__riscv_save_8",
188 "__riscv_save_9",
189 "__riscv_save_10",
190 "__riscv_save_11",
191 "__riscv_save_12"
192 };
193
194 int LibCallID = getLibCallID(MF, CSI);
195 if (LibCallID == -1)
196 return nullptr;
197 return SpillLibCalls[LibCallID];
198 }
199
200 // Get the name of the libcall used for restoring callee saved registers.
201 // If this function will not use save/restore libcalls, then return a nullptr.
202 static const char *
getRestoreLibCallName(const MachineFunction & MF,const std::vector<CalleeSavedInfo> & CSI)203 getRestoreLibCallName(const MachineFunction &MF,
204 const std::vector<CalleeSavedInfo> &CSI) {
205 static const char *const RestoreLibCalls[] = {
206 "__riscv_restore_0",
207 "__riscv_restore_1",
208 "__riscv_restore_2",
209 "__riscv_restore_3",
210 "__riscv_restore_4",
211 "__riscv_restore_5",
212 "__riscv_restore_6",
213 "__riscv_restore_7",
214 "__riscv_restore_8",
215 "__riscv_restore_9",
216 "__riscv_restore_10",
217 "__riscv_restore_11",
218 "__riscv_restore_12"
219 };
220
221 int LibCallID = getLibCallID(MF, CSI);
222 if (LibCallID == -1)
223 return nullptr;
224 return RestoreLibCalls[LibCallID];
225 }
226
227 // Return true if the specified function should have a dedicated frame
228 // pointer register. This is true if frame pointer elimination is
229 // disabled, if it needs dynamic stack realignment, if the function has
230 // variable sized allocas, or if the frame address is taken.
hasFP(const MachineFunction & MF) const231 bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const {
232 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
233
234 const MachineFrameInfo &MFI = MF.getFrameInfo();
235 return MF.getTarget().Options.DisableFramePointerElim(MF) ||
236 RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
237 MFI.isFrameAddressTaken();
238 }
239
hasBP(const MachineFunction & MF) const240 bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const {
241 const MachineFrameInfo &MFI = MF.getFrameInfo();
242 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
243
244 // If we do not reserve stack space for outgoing arguments in prologue,
245 // we will adjust the stack pointer before call instruction. After the
246 // adjustment, we can not use SP to access the stack objects for the
247 // arguments. Instead, use BP to access these stack objects.
248 return (MFI.hasVarSizedObjects() ||
249 (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() ||
250 MFI.getMaxCallFrameSize() != 0))) &&
251 TRI->hasStackRealignment(MF);
252 }
253
254 // Determines the size of the frame and maximum call frame size.
determineFrameLayout(MachineFunction & MF) const255 void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
256 MachineFrameInfo &MFI = MF.getFrameInfo();
257 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
258
259 // Get the number of bytes to allocate from the FrameInfo.
260 uint64_t FrameSize = MFI.getStackSize();
261
262 // Get the alignment.
263 Align StackAlign = getStackAlign();
264
265 // Make sure the frame is aligned.
266 FrameSize = alignTo(FrameSize, StackAlign);
267
268 // Update frame info.
269 MFI.setStackSize(FrameSize);
270
271 // When using SP or BP to access stack objects, we may require extra padding
272 // to ensure the bottom of the RVV stack is correctly aligned within the main
273 // stack. We calculate this as the amount required to align the scalar local
274 // variable section up to the RVV alignment.
275 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
276 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
277 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
278 RVFI->getVarArgsSaveSize();
279 if (auto RVVPadding =
280 offsetToAlignment(ScalarLocalVarSize, RVFI->getRVVStackAlign()))
281 RVFI->setRVVPadding(RVVPadding);
282 }
283 }
284
285 // Returns the stack size including RVV padding (when required), rounded back
286 // up to the required stack alignment.
getStackSizeWithRVVPadding(const MachineFunction & MF) const287 uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding(
288 const MachineFunction &MF) const {
289 const MachineFrameInfo &MFI = MF.getFrameInfo();
290 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
291 return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign());
292 }
293
294 // Returns the register used to hold the frame pointer.
getFPReg(const RISCVSubtarget & STI)295 static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; }
296
297 // Returns the register used to hold the stack pointer.
getSPReg(const RISCVSubtarget & STI)298 static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; }
299
300 static SmallVector<CalleeSavedInfo, 8>
getNonLibcallCSI(const MachineFunction & MF,const std::vector<CalleeSavedInfo> & CSI)301 getNonLibcallCSI(const MachineFunction &MF,
302 const std::vector<CalleeSavedInfo> &CSI) {
303 const MachineFrameInfo &MFI = MF.getFrameInfo();
304 SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
305
306 for (auto &CS : CSI) {
307 int FI = CS.getFrameIdx();
308 if (FI >= 0 && MFI.getStackID(FI) == TargetStackID::Default)
309 NonLibcallCSI.push_back(CS);
310 }
311
312 return NonLibcallCSI;
313 }
314
adjustStackForRVV(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,int64_t Amount,MachineInstr::MIFlag Flag) const315 void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF,
316 MachineBasicBlock &MBB,
317 MachineBasicBlock::iterator MBBI,
318 const DebugLoc &DL, int64_t Amount,
319 MachineInstr::MIFlag Flag) const {
320 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
321
322 const Register SPReg = getSPReg(STI);
323
324 // Optimize compile time offset case
325 StackOffset Offset = StackOffset::getScalable(Amount);
326 if (STI.getRealMinVLen() == STI.getRealMaxVLen()) {
327 // 1. Multiply the number of v-slots by the (constant) length of register
328 const int64_t VLENB = STI.getRealMinVLen() / 8;
329 assert(Amount % 8 == 0 &&
330 "Reserve the stack by the multiple of one vector size.");
331 const int64_t NumOfVReg = Amount / 8;
332 const int64_t FixedOffset = NumOfVReg * VLENB;
333 if (!isInt<32>(FixedOffset)) {
334 report_fatal_error(
335 "Frame size outside of the signed 32-bit range not supported");
336 }
337 Offset = StackOffset::getFixed(FixedOffset);
338 }
339
340 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
341 // We must keep the stack pointer aligned through any intermediate
342 // updates.
343 RI.adjustReg(MBB, MBBI, DL, SPReg, SPReg, Offset,
344 Flag, getStackAlign());
345 }
346
createDefCFAExpression(const TargetRegisterInfo & TRI,Register Reg,uint64_t FixedOffset,uint64_t ScalableOffset)347 static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
348 Register Reg,
349 uint64_t FixedOffset,
350 uint64_t ScalableOffset) {
351 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
352 SmallString<64> Expr;
353 std::string CommentBuffer;
354 llvm::raw_string_ostream Comment(CommentBuffer);
355 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
356 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
357 Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
358 Expr.push_back(0);
359 if (Reg == RISCV::X2)
360 Comment << "sp";
361 else
362 Comment << printReg(Reg, &TRI);
363
364 uint8_t buffer[16];
365 if (FixedOffset) {
366 Expr.push_back(dwarf::DW_OP_consts);
367 Expr.append(buffer, buffer + encodeSLEB128(FixedOffset, buffer));
368 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
369 Comment << " + " << FixedOffset;
370 }
371
372 Expr.push_back((uint8_t)dwarf::DW_OP_consts);
373 Expr.append(buffer, buffer + encodeSLEB128(ScalableOffset, buffer));
374
375 unsigned DwarfVlenb = TRI.getDwarfRegNum(RISCV::VLENB, true);
376 Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
377 Expr.append(buffer, buffer + encodeULEB128(DwarfVlenb, buffer));
378 Expr.push_back(0);
379
380 Expr.push_back((uint8_t)dwarf::DW_OP_mul);
381 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
382
383 Comment << " + " << ScalableOffset << " * vlenb";
384
385 SmallString<64> DefCfaExpr;
386 DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
387 DefCfaExpr.append(buffer, buffer + encodeULEB128(Expr.size(), buffer));
388 DefCfaExpr.append(Expr.str());
389
390 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
391 Comment.str());
392 }
393
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const394 void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
395 MachineBasicBlock &MBB) const {
396 MachineFrameInfo &MFI = MF.getFrameInfo();
397 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
398 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
399 const RISCVInstrInfo *TII = STI.getInstrInfo();
400 MachineBasicBlock::iterator MBBI = MBB.begin();
401
402 Register FPReg = getFPReg(STI);
403 Register SPReg = getSPReg(STI);
404 Register BPReg = RISCVABI::getBPReg();
405
406 // Debug location must be unknown since the first debug location is used
407 // to determine the end of the prologue.
408 DebugLoc DL;
409
410 // All calls are tail calls in GHC calling conv, and functions have no
411 // prologue/epilogue.
412 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
413 return;
414
415 // Emit prologue for shadow call stack.
416 emitSCSPrologue(MF, MBB, MBBI, DL);
417
418 // Since spillCalleeSavedRegisters may have inserted a libcall, skip past
419 // any instructions marked as FrameSetup
420 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
421 ++MBBI;
422
423 // Determine the correct frame layout
424 determineFrameLayout(MF);
425
426 // If libcalls are used to spill and restore callee-saved registers, the frame
427 // has two sections; the opaque section managed by the libcalls, and the
428 // section managed by MachineFrameInfo which can also hold callee saved
429 // registers in fixed stack slots, both of which have negative frame indices.
430 // This gets even more complicated when incoming arguments are passed via the
431 // stack, as these too have negative frame indices. An example is detailed
432 // below:
433 //
434 // | incoming arg | <- FI[-3]
435 // | libcallspill |
436 // | calleespill | <- FI[-2]
437 // | calleespill | <- FI[-1]
438 // | this_frame | <- FI[0]
439 //
440 // For negative frame indices, the offset from the frame pointer will differ
441 // depending on which of these groups the frame index applies to.
442 // The following calculates the correct offset knowing the number of callee
443 // saved registers spilt by the two methods.
444 if (int LibCallRegs = getLibCallID(MF, MFI.getCalleeSavedInfo()) + 1) {
445 // Calculate the size of the frame managed by the libcall. The libcalls are
446 // implemented such that the stack will always be 16 byte aligned.
447 unsigned LibCallFrameSize = alignTo((STI.getXLen() / 8) * LibCallRegs, 16);
448 RVFI->setLibCallStackSize(LibCallFrameSize);
449 }
450
451 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
452 // investigation. Get the number of bytes to allocate from the FrameInfo.
453 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
454 uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize();
455 uint64_t RVVStackSize = RVFI->getRVVStackSize();
456
457 // Early exit if there is no need to allocate on the stack
458 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
459 return;
460
461 // If the stack pointer has been marked as reserved, then produce an error if
462 // the frame requires stack allocation
463 if (STI.isRegisterReservedByUser(SPReg))
464 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
465 MF.getFunction(), "Stack pointer required, but has been reserved."});
466
467 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
468 // Split the SP adjustment to reduce the offsets of callee saved spill.
469 if (FirstSPAdjustAmount) {
470 StackSize = FirstSPAdjustAmount;
471 RealStackSize = FirstSPAdjustAmount;
472 }
473
474 // Allocate space on the stack if necessary.
475 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-StackSize),
476 MachineInstr::FrameSetup, getStackAlign());
477
478 // Emit ".cfi_def_cfa_offset RealStackSize"
479 unsigned CFIIndex = MF.addFrameInst(
480 MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize));
481 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
482 .addCFIIndex(CFIIndex)
483 .setMIFlag(MachineInstr::FrameSetup);
484
485 const auto &CSI = MFI.getCalleeSavedInfo();
486
487 // The frame pointer is callee-saved, and code has been generated for us to
488 // save it to the stack. We need to skip over the storing of callee-saved
489 // registers as the frame pointer must be modified after it has been saved
490 // to the stack, not before.
491 // FIXME: assumes exactly one instruction is used to save each callee-saved
492 // register.
493 std::advance(MBBI, getNonLibcallCSI(MF, CSI).size());
494
495 // Iterate over list of callee-saved registers and emit .cfi_offset
496 // directives.
497 for (const auto &Entry : CSI) {
498 int FrameIdx = Entry.getFrameIdx();
499 int64_t Offset;
500 // Offsets for objects with fixed locations (IE: those saved by libcall) are
501 // simply calculated from the frame index.
502 if (FrameIdx < 0)
503 Offset = FrameIdx * (int64_t) STI.getXLen() / 8;
504 else
505 Offset = MFI.getObjectOffset(Entry.getFrameIdx()) -
506 RVFI->getLibCallStackSize();
507 Register Reg = Entry.getReg();
508 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
509 nullptr, RI->getDwarfRegNum(Reg, true), Offset));
510 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
511 .addCFIIndex(CFIIndex)
512 .setMIFlag(MachineInstr::FrameSetup);
513 }
514
515 // Generate new FP.
516 if (hasFP(MF)) {
517 if (STI.isRegisterReservedByUser(FPReg))
518 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
519 MF.getFunction(), "Frame pointer required, but has been reserved."});
520 // The frame pointer does need to be reserved from register allocation.
521 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
522
523 RI->adjustReg(MBB, MBBI, DL, FPReg, SPReg,
524 StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
525 MachineInstr::FrameSetup, getStackAlign());
526
527 // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
528 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
529 nullptr, RI->getDwarfRegNum(FPReg, true), RVFI->getVarArgsSaveSize()));
530 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
531 .addCFIIndex(CFIIndex)
532 .setMIFlag(MachineInstr::FrameSetup);
533 }
534
535 // Emit the second SP adjustment after saving callee saved registers.
536 if (FirstSPAdjustAmount) {
537 uint64_t SecondSPAdjustAmount =
538 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
539 assert(SecondSPAdjustAmount > 0 &&
540 "SecondSPAdjustAmount should be greater than zero");
541 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
542 StackOffset::getFixed(-SecondSPAdjustAmount),
543 MachineInstr::FrameSetup, getStackAlign());
544
545 // If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0",
546 // don't emit an sp-based .cfi_def_cfa_offset
547 if (!hasFP(MF)) {
548 // Emit ".cfi_def_cfa_offset StackSize"
549 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(
550 nullptr, getStackSizeWithRVVPadding(MF)));
551 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
552 .addCFIIndex(CFIIndex)
553 .setMIFlag(MachineInstr::FrameSetup);
554 }
555 }
556
557 if (RVVStackSize) {
558 adjustStackForRVV(MF, MBB, MBBI, DL, -RVVStackSize,
559 MachineInstr::FrameSetup);
560 if (!hasFP(MF)) {
561 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
562 unsigned CFIIndex = MF.addFrameInst(createDefCFAExpression(
563 *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
564 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
565 .addCFIIndex(CFIIndex)
566 .setMIFlag(MachineInstr::FrameSetup);
567 }
568 }
569
570 if (hasFP(MF)) {
571 // Realign Stack
572 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
573 if (RI->hasStackRealignment(MF)) {
574 Align MaxAlignment = MFI.getMaxAlign();
575
576 const RISCVInstrInfo *TII = STI.getInstrInfo();
577 if (isInt<12>(-(int)MaxAlignment.value())) {
578 BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg)
579 .addReg(SPReg)
580 .addImm(-(int)MaxAlignment.value())
581 .setMIFlag(MachineInstr::FrameSetup);
582 } else {
583 unsigned ShiftAmount = Log2(MaxAlignment);
584 Register VR =
585 MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);
586 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR)
587 .addReg(SPReg)
588 .addImm(ShiftAmount)
589 .setMIFlag(MachineInstr::FrameSetup);
590 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg)
591 .addReg(VR)
592 .addImm(ShiftAmount)
593 .setMIFlag(MachineInstr::FrameSetup);
594 }
595 // FP will be used to restore the frame in the epilogue, so we need
596 // another base register BP to record SP after re-alignment. SP will
597 // track the current stack after allocating variable sized objects.
598 if (hasBP(MF)) {
599 // move BP, SP
600 BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg)
601 .addReg(SPReg)
602 .addImm(0)
603 .setMIFlag(MachineInstr::FrameSetup);
604 }
605 }
606 }
607 }
608
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const609 void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
610 MachineBasicBlock &MBB) const {
611 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
612 MachineFrameInfo &MFI = MF.getFrameInfo();
613 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
614 Register FPReg = getFPReg(STI);
615 Register SPReg = getSPReg(STI);
616
617 // All calls are tail calls in GHC calling conv, and functions have no
618 // prologue/epilogue.
619 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
620 return;
621
622 // Get the insert location for the epilogue. If there were no terminators in
623 // the block, get the last instruction.
624 MachineBasicBlock::iterator MBBI = MBB.end();
625 DebugLoc DL;
626 if (!MBB.empty()) {
627 MBBI = MBB.getLastNonDebugInstr();
628 if (MBBI != MBB.end())
629 DL = MBBI->getDebugLoc();
630
631 MBBI = MBB.getFirstTerminator();
632
633 // If callee-saved registers are saved via libcall, place stack adjustment
634 // before this call.
635 while (MBBI != MBB.begin() &&
636 std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy))
637 --MBBI;
638 }
639
640 const auto &CSI = getNonLibcallCSI(MF, MFI.getCalleeSavedInfo());
641
642 // Skip to before the restores of callee-saved registers
643 // FIXME: assumes exactly one instruction is used to restore each
644 // callee-saved register.
645 auto LastFrameDestroy = MBBI;
646 if (!CSI.empty())
647 LastFrameDestroy = std::prev(MBBI, CSI.size());
648
649 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
650 uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize();
651 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
652 uint64_t RVVStackSize = RVFI->getRVVStackSize();
653
654 // Restore the stack pointer using the value of the frame pointer. Only
655 // necessary if the stack pointer was modified, meaning the stack size is
656 // unknown.
657 //
658 // In order to make sure the stack point is right through the EH region,
659 // we also need to restore stack pointer from the frame pointer if we
660 // don't preserve stack space within prologue/epilogue for outgoing variables,
661 // normally it's just checking the variable sized object is present or not
662 // is enough, but we also don't preserve that at prologue/epilogue when
663 // have vector objects in stack.
664 if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
665 !hasReservedCallFrame(MF)) {
666 assert(hasFP(MF) && "frame pointer should not have been eliminated");
667 RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, FPReg,
668 StackOffset::getFixed(-FPOffset),
669 MachineInstr::FrameDestroy, getStackAlign());
670 } else {
671 if (RVVStackSize)
672 adjustStackForRVV(MF, MBB, LastFrameDestroy, DL, RVVStackSize,
673 MachineInstr::FrameDestroy);
674 }
675
676 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
677 if (FirstSPAdjustAmount) {
678 uint64_t SecondSPAdjustAmount =
679 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
680 assert(SecondSPAdjustAmount > 0 &&
681 "SecondSPAdjustAmount should be greater than zero");
682
683 RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg,
684 StackOffset::getFixed(SecondSPAdjustAmount),
685 MachineInstr::FrameDestroy, getStackAlign());
686 }
687
688 if (FirstSPAdjustAmount)
689 StackSize = FirstSPAdjustAmount;
690
691 // Deallocate stack
692 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
693 MachineInstr::FrameDestroy, getStackAlign());
694
695 // Emit epilogue for shadow call stack.
696 emitSCSEpilogue(MF, MBB, MBBI, DL);
697 }
698
699 StackOffset
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const700 RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
701 Register &FrameReg) const {
702 const MachineFrameInfo &MFI = MF.getFrameInfo();
703 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
704 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
705
706 // Callee-saved registers should be referenced relative to the stack
707 // pointer (positive offset), otherwise use the frame pointer (negative
708 // offset).
709 const auto &CSI = getNonLibcallCSI(MF, MFI.getCalleeSavedInfo());
710 int MinCSFI = 0;
711 int MaxCSFI = -1;
712 StackOffset Offset;
713 auto StackID = MFI.getStackID(FI);
714
715 assert((StackID == TargetStackID::Default ||
716 StackID == TargetStackID::ScalableVector) &&
717 "Unexpected stack ID for the frame object.");
718 if (StackID == TargetStackID::Default) {
719 Offset =
720 StackOffset::getFixed(MFI.getObjectOffset(FI) - getOffsetOfLocalArea() +
721 MFI.getOffsetAdjustment());
722 } else if (StackID == TargetStackID::ScalableVector) {
723 Offset = StackOffset::getScalable(MFI.getObjectOffset(FI));
724 }
725
726 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
727
728 if (CSI.size()) {
729 MinCSFI = CSI[0].getFrameIdx();
730 MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
731 }
732
733 if (FI >= MinCSFI && FI <= MaxCSFI) {
734 FrameReg = RISCV::X2;
735
736 if (FirstSPAdjustAmount)
737 Offset += StackOffset::getFixed(FirstSPAdjustAmount);
738 else
739 Offset += StackOffset::getFixed(getStackSizeWithRVVPadding(MF));
740 return Offset;
741 }
742
743 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(FI)) {
744 // If the stack was realigned, the frame pointer is set in order to allow
745 // SP to be restored, so we need another base register to record the stack
746 // after realignment.
747 // |--------------------------| -- <-- FP
748 // | callee-allocated save | | <----|
749 // | area for register varargs| | |
750 // |--------------------------| | |
751 // | callee-saved registers | | |
752 // |--------------------------| -- |
753 // | realignment (the size of | | |
754 // | this area is not counted | | |
755 // | in MFI.getStackSize()) | | |
756 // |--------------------------| -- |-- MFI.getStackSize()
757 // | RVV alignment padding | | |
758 // | (not counted in | | |
759 // | MFI.getStackSize() but | | |
760 // | counted in | | |
761 // | RVFI.getRVVStackSize()) | | |
762 // |--------------------------| -- |
763 // | RVV objects | | |
764 // | (not counted in | | |
765 // | MFI.getStackSize()) | | |
766 // |--------------------------| -- |
767 // | padding before RVV | | |
768 // | (not counted in | | |
769 // | MFI.getStackSize() or in | | |
770 // | RVFI.getRVVStackSize()) | | |
771 // |--------------------------| -- |
772 // | scalar local variables | | <----'
773 // |--------------------------| -- <-- BP (if var sized objects present)
774 // | VarSize objects | |
775 // |--------------------------| -- <-- SP
776 if (hasBP(MF)) {
777 FrameReg = RISCVABI::getBPReg();
778 } else {
779 // VarSize objects must be empty in this case!
780 assert(!MFI.hasVarSizedObjects());
781 FrameReg = RISCV::X2;
782 }
783 } else {
784 FrameReg = RI->getFrameRegister(MF);
785 }
786
787 if (FrameReg == getFPReg(STI)) {
788 Offset += StackOffset::getFixed(RVFI->getVarArgsSaveSize());
789 if (FI >= 0)
790 Offset -= StackOffset::getFixed(RVFI->getLibCallStackSize());
791 // When using FP to access scalable vector objects, we need to minus
792 // the frame size.
793 //
794 // |--------------------------| -- <-- FP
795 // | callee-allocated save | |
796 // | area for register varargs| |
797 // |--------------------------| |
798 // | callee-saved registers | |
799 // |--------------------------| | MFI.getStackSize()
800 // | scalar local variables | |
801 // |--------------------------| -- (Offset of RVV objects is from here.)
802 // | RVV objects |
803 // |--------------------------|
804 // | VarSize objects |
805 // |--------------------------| <-- SP
806 if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
807 assert(!RI->hasStackRealignment(MF) &&
808 "Can't index across variable sized realign");
809 // We don't expect any extra RVV alignment padding, as the stack size
810 // and RVV object sections should be correct aligned in their own
811 // right.
812 assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) &&
813 "Inconsistent stack layout");
814 Offset -= StackOffset::getFixed(MFI.getStackSize());
815 }
816 return Offset;
817 }
818
819 // This case handles indexing off both SP and BP.
820 // If indexing off SP, there must not be any var sized objects
821 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
822
823 // When using SP to access frame objects, we need to add RVV stack size.
824 //
825 // |--------------------------| -- <-- FP
826 // | callee-allocated save | | <----|
827 // | area for register varargs| | |
828 // |--------------------------| | |
829 // | callee-saved registers | | |
830 // |--------------------------| -- |
831 // | RVV alignment padding | | |
832 // | (not counted in | | |
833 // | MFI.getStackSize() but | | |
834 // | counted in | | |
835 // | RVFI.getRVVStackSize()) | | |
836 // |--------------------------| -- |
837 // | RVV objects | | |-- MFI.getStackSize()
838 // | (not counted in | | |
839 // | MFI.getStackSize()) | | |
840 // |--------------------------| -- |
841 // | padding before RVV | | |
842 // | (not counted in | | |
843 // | MFI.getStackSize()) | | |
844 // |--------------------------| -- |
845 // | scalar local variables | | <----'
846 // |--------------------------| -- <-- BP (if var sized objects present)
847 // | VarSize objects | |
848 // |--------------------------| -- <-- SP
849 //
850 // The total amount of padding surrounding RVV objects is described by
851 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
852 // objects to the required alignment.
853 if (MFI.getStackID(FI) == TargetStackID::Default) {
854 if (MFI.isFixedObjectIndex(FI)) {
855 assert(!RI->hasStackRealignment(MF) &&
856 "Can't index across variable sized realign");
857 Offset += StackOffset::get(getStackSizeWithRVVPadding(MF) +
858 RVFI->getLibCallStackSize(),
859 RVFI->getRVVStackSize());
860 } else {
861 Offset += StackOffset::getFixed(MFI.getStackSize());
862 }
863 } else if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
864 // Ensure the base of the RVV stack is correctly aligned: add on the
865 // alignment padding.
866 int ScalarLocalVarSize =
867 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
868 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
869 Offset += StackOffset::get(ScalarLocalVarSize, RVFI->getRVVStackSize());
870 }
871 return Offset;
872 }
873
determineCalleeSaves(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const874 void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
875 BitVector &SavedRegs,
876 RegScavenger *RS) const {
877 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
878 // Unconditionally spill RA and FP only if the function uses a frame
879 // pointer.
880 if (hasFP(MF)) {
881 SavedRegs.set(RISCV::X1);
882 SavedRegs.set(RISCV::X8);
883 }
884 // Mark BP as used if function has dedicated base pointer.
885 if (hasBP(MF))
886 SavedRegs.set(RISCVABI::getBPReg());
887
888 // If interrupt is enabled and there are calls in the handler,
889 // unconditionally save all Caller-saved registers and
890 // all FP registers, regardless whether they are used.
891 MachineFrameInfo &MFI = MF.getFrameInfo();
892
893 if (MF.getFunction().hasFnAttribute("interrupt") && MFI.hasCalls()) {
894
895 static const MCPhysReg CSRegs[] = { RISCV::X1, /* ra */
896 RISCV::X5, RISCV::X6, RISCV::X7, /* t0-t2 */
897 RISCV::X10, RISCV::X11, /* a0-a1, a2-a7 */
898 RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17,
899 RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31, 0 /* t3-t6 */
900 };
901
902 for (unsigned i = 0; CSRegs[i]; ++i)
903 SavedRegs.set(CSRegs[i]);
904
905 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtF()) {
906
907 // If interrupt is enabled, this list contains all FP registers.
908 const MCPhysReg * Regs = MF.getRegInfo().getCalleeSavedRegs();
909
910 for (unsigned i = 0; Regs[i]; ++i)
911 if (RISCV::FPR16RegClass.contains(Regs[i]) ||
912 RISCV::FPR32RegClass.contains(Regs[i]) ||
913 RISCV::FPR64RegClass.contains(Regs[i]))
914 SavedRegs.set(Regs[i]);
915 }
916 }
917 }
918
919 std::pair<int64_t, Align>
assignRVVStackObjectOffsets(MachineFunction & MF) const920 RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
921 MachineFrameInfo &MFI = MF.getFrameInfo();
922 // Create a buffer of RVV objects to allocate.
923 SmallVector<int, 8> ObjectsToAllocate;
924 for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
925 unsigned StackID = MFI.getStackID(I);
926 if (StackID != TargetStackID::ScalableVector)
927 continue;
928 if (MFI.isDeadObjectIndex(I))
929 continue;
930
931 ObjectsToAllocate.push_back(I);
932 }
933
934 // The minimum alignment is 16 bytes.
935 Align RVVStackAlign(16);
936 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
937
938 if (!ST.hasVInstructions()) {
939 assert(ObjectsToAllocate.empty() &&
940 "Can't allocate scalable-vector objects without V instructions");
941 return std::make_pair(0, RVVStackAlign);
942 }
943
944 // Allocate all RVV locals and spills
945 int64_t Offset = 0;
946 for (int FI : ObjectsToAllocate) {
947 // ObjectSize in bytes.
948 int64_t ObjectSize = MFI.getObjectSize(FI);
949 auto ObjectAlign = std::max(Align(8), MFI.getObjectAlign(FI));
950 // If the data type is the fractional vector type, reserve one vector
951 // register for it.
952 if (ObjectSize < 8)
953 ObjectSize = 8;
954 Offset = alignTo(Offset + ObjectSize, ObjectAlign);
955 MFI.setObjectOffset(FI, -Offset);
956 // Update the maximum alignment of the RVV stack section
957 RVVStackAlign = std::max(RVVStackAlign, ObjectAlign);
958 }
959
960 // Ensure the alignment of the RVV stack. Since we want the most-aligned
961 // object right at the bottom (i.e., any padding at the top of the frame),
962 // readjust all RVV objects down by the alignment padding.
963 uint64_t StackSize = Offset;
964 if (auto AlignmentPadding = offsetToAlignment(StackSize, RVVStackAlign)) {
965 StackSize += AlignmentPadding;
966 for (int FI : ObjectsToAllocate)
967 MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding);
968 }
969
970 return std::make_pair(StackSize, RVVStackAlign);
971 }
972
getScavSlotsNumForRVV(MachineFunction & MF)973 static unsigned getScavSlotsNumForRVV(MachineFunction &MF) {
974 // For RVV spill, scalable stack offsets computing requires up to two scratch
975 // registers
976 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
977
978 // For RVV spill, non-scalable stack offsets computing requires up to one
979 // scratch register.
980 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
981
982 // ADDI instruction's destination register can be used for computing
983 // offsets. So Scalable stack offsets require up to one scratch register.
984 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
985
986 static constexpr unsigned MaxScavSlotsNumKnown =
987 std::max({ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
988 ScavSlotsNumRVVSpillNonScalableObject});
989
990 unsigned MaxScavSlotsNum = 0;
991 if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
992 return false;
993 for (const MachineBasicBlock &MBB : MF)
994 for (const MachineInstr &MI : MBB) {
995 bool IsRVVSpill = RISCV::isRVVSpill(MI);
996 for (auto &MO : MI.operands()) {
997 if (!MO.isFI())
998 continue;
999 bool IsScalableVectorID = MF.getFrameInfo().getStackID(MO.getIndex()) ==
1000 TargetStackID::ScalableVector;
1001 if (IsRVVSpill) {
1002 MaxScavSlotsNum = std::max(
1003 MaxScavSlotsNum, IsScalableVectorID
1004 ? ScavSlotsNumRVVSpillScalableObject
1005 : ScavSlotsNumRVVSpillNonScalableObject);
1006 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
1007 MaxScavSlotsNum =
1008 std::max(MaxScavSlotsNum, ScavSlotsADDIScalableObject);
1009 }
1010 }
1011 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
1012 return MaxScavSlotsNumKnown;
1013 }
1014 return MaxScavSlotsNum;
1015 }
1016
hasRVVFrameObject(const MachineFunction & MF)1017 static bool hasRVVFrameObject(const MachineFunction &MF) {
1018 // Originally, the function will scan all the stack objects to check whether
1019 // if there is any scalable vector object on the stack or not. However, it
1020 // causes errors in the register allocator. In issue 53016, it returns false
1021 // before RA because there is no RVV stack objects. After RA, it returns true
1022 // because there are spilling slots for RVV values during RA. It will not
1023 // reserve BP during register allocation and generate BP access in the PEI
1024 // pass due to the inconsistent behavior of the function.
1025 //
1026 // The function is changed to use hasVInstructions() as the return value. It
1027 // is not precise, but it can make the register allocation correct.
1028 //
1029 // FIXME: Find a better way to make the decision or revisit the solution in
1030 // D103622.
1031 //
1032 // Refer to https://github.com/llvm/llvm-project/issues/53016.
1033 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
1034 }
1035
estimateFunctionSizeInBytes(const MachineFunction & MF,const RISCVInstrInfo & TII)1036 static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF,
1037 const RISCVInstrInfo &TII) {
1038 unsigned FnSize = 0;
1039 for (auto &MBB : MF) {
1040 for (auto &MI : MBB) {
1041 // Far branches over 20-bit offset will be relaxed in branch relaxation
1042 // pass. In the worst case, conditional branches will be relaxed into
1043 // the following instruction sequence. Unconditional branches are
1044 // relaxed in the same way, with the exception that there is no first
1045 // branch instruction.
1046 //
1047 // foo
1048 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
1049 // sd s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1050 // jump .restore, s11 # 8 bytes
1051 // .rev_cond
1052 // bar
1053 // j .dest_bb # 4 bytes, or 2 bytes in RVC
1054 // .restore:
1055 // ld s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1056 // .dest:
1057 // baz
1058 if (MI.isConditionalBranch())
1059 FnSize += TII.getInstSizeInBytes(MI);
1060 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
1061 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtC())
1062 FnSize += 2 + 8 + 2 + 2;
1063 else
1064 FnSize += 4 + 8 + 4 + 4;
1065 continue;
1066 }
1067
1068 FnSize += TII.getInstSizeInBytes(MI);
1069 }
1070 }
1071 return FnSize;
1072 }
1073
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const1074 void RISCVFrameLowering::processFunctionBeforeFrameFinalized(
1075 MachineFunction &MF, RegScavenger *RS) const {
1076 const RISCVRegisterInfo *RegInfo =
1077 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
1078 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
1079 MachineFrameInfo &MFI = MF.getFrameInfo();
1080 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1081 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1082
1083 int64_t RVVStackSize;
1084 Align RVVStackAlign;
1085 std::tie(RVVStackSize, RVVStackAlign) = assignRVVStackObjectOffsets(MF);
1086
1087 RVFI->setRVVStackSize(RVVStackSize);
1088 RVFI->setRVVStackAlign(RVVStackAlign);
1089
1090 if (hasRVVFrameObject(MF)) {
1091 // Ensure the entire stack is aligned to at least the RVV requirement: some
1092 // scalable-vector object alignments are not considered by the
1093 // target-independent code.
1094 MFI.ensureMaxAlignment(RVVStackAlign);
1095 }
1096
1097 unsigned ScavSlotsNum = 0;
1098
1099 // estimateStackSize has been observed to under-estimate the final stack
1100 // size, so give ourselves wiggle-room by checking for stack size
1101 // representable an 11-bit signed field rather than 12-bits.
1102 if (!isInt<11>(MFI.estimateStackSize(MF)))
1103 ScavSlotsNum = 1;
1104
1105 // Far branches over 20-bit offset require a spill slot for scratch register.
1106 bool IsLargeFunction = !isInt<20>(estimateFunctionSizeInBytes(MF, *TII));
1107 if (IsLargeFunction)
1108 ScavSlotsNum = std::max(ScavSlotsNum, 1u);
1109
1110 // RVV loads & stores have no capacity to hold the immediate address offsets
1111 // so we must always reserve an emergency spill slot if the MachineFunction
1112 // contains any RVV spills.
1113 ScavSlotsNum = std::max(ScavSlotsNum, getScavSlotsNumForRVV(MF));
1114
1115 for (unsigned I = 0; I < ScavSlotsNum; I++) {
1116 int FI = MFI.CreateStackObject(RegInfo->getSpillSize(*RC),
1117 RegInfo->getSpillAlign(*RC), false);
1118 RS->addScavengingFrameIndex(FI);
1119
1120 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
1121 RVFI->setBranchRelaxationScratchFrameIndex(FI);
1122 }
1123
1124 if (MFI.getCalleeSavedInfo().empty() || RVFI->useSaveRestoreLibCalls(MF)) {
1125 RVFI->setCalleeSavedStackSize(0);
1126 return;
1127 }
1128
1129 unsigned Size = 0;
1130 for (const auto &Info : MFI.getCalleeSavedInfo()) {
1131 int FrameIdx = Info.getFrameIdx();
1132 if (MFI.getStackID(FrameIdx) != TargetStackID::Default)
1133 continue;
1134
1135 Size += MFI.getObjectSize(FrameIdx);
1136 }
1137 RVFI->setCalleeSavedStackSize(Size);
1138 }
1139
1140 // Not preserve stack space within prologue for outgoing variables when the
1141 // function contains variable size objects or there are vector objects accessed
1142 // by the frame pointer.
1143 // Let eliminateCallFramePseudoInstr preserve stack space for it.
hasReservedCallFrame(const MachineFunction & MF) const1144 bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
1145 return !MF.getFrameInfo().hasVarSizedObjects() &&
1146 !(hasFP(MF) && hasRVVFrameObject(MF));
1147 }
1148
1149 // Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MI) const1150 MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
1151 MachineFunction &MF, MachineBasicBlock &MBB,
1152 MachineBasicBlock::iterator MI) const {
1153 Register SPReg = RISCV::X2;
1154 DebugLoc DL = MI->getDebugLoc();
1155
1156 if (!hasReservedCallFrame(MF)) {
1157 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
1158 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
1159 // pointer. This is necessary when there is a variable length stack
1160 // allocation (e.g. alloca), which means it's not possible to allocate
1161 // space for outgoing arguments from within the function prologue.
1162 int64_t Amount = MI->getOperand(0).getImm();
1163
1164 if (Amount != 0) {
1165 // Ensure the stack remains aligned after adjustment.
1166 Amount = alignSPAdjust(Amount);
1167
1168 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
1169 Amount = -Amount;
1170
1171 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
1172 RI.adjustReg(MBB, MI, DL, SPReg, SPReg, StackOffset::getFixed(Amount),
1173 MachineInstr::NoFlags, getStackAlign());
1174 }
1175 }
1176
1177 return MBB.erase(MI);
1178 }
1179
1180 // We would like to split the SP adjustment to reduce prologue/epilogue
1181 // as following instructions. In this way, the offset of the callee saved
1182 // register could fit in a single store.
1183 // add sp,sp,-2032
1184 // sw ra,2028(sp)
1185 // sw s0,2024(sp)
1186 // sw s1,2020(sp)
1187 // sw s3,2012(sp)
1188 // sw s4,2008(sp)
1189 // add sp,sp,-64
1190 uint64_t
getFirstSPAdjustAmount(const MachineFunction & MF) const1191 RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
1192 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1193 const MachineFrameInfo &MFI = MF.getFrameInfo();
1194 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
1195 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
1196
1197 // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved
1198 // registers will be pushed by the save-restore libcalls, so we don't have to
1199 // split the SP adjustment in this case.
1200 if (RVFI->getLibCallStackSize())
1201 return 0;
1202
1203 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
1204 // 12-bit and there exists a callee-saved register needing to be pushed.
1205 if (!isInt<12>(StackSize) && (CSI.size() > 0)) {
1206 // FirstSPAdjustAmount is chosen as (2048 - StackAlign) because 2048 will
1207 // cause sp = sp + 2048 in the epilogue to be split into multiple
1208 // instructions. Offsets smaller than 2048 can fit in a single load/store
1209 // instruction, and we have to stick with the stack alignment. 2048 has
1210 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
1211 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
1212 return 2048 - getStackAlign().value();
1213 }
1214 return 0;
1215 }
1216
spillCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const1217 bool RISCVFrameLowering::spillCalleeSavedRegisters(
1218 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1219 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1220 if (CSI.empty())
1221 return true;
1222
1223 MachineFunction *MF = MBB.getParent();
1224 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1225 DebugLoc DL;
1226 if (MI != MBB.end() && !MI->isDebugInstr())
1227 DL = MI->getDebugLoc();
1228
1229 const char *SpillLibCall = getSpillLibCallName(*MF, CSI);
1230 if (SpillLibCall) {
1231 // Add spill libcall via non-callee-saved register t0.
1232 BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5)
1233 .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL)
1234 .setMIFlag(MachineInstr::FrameSetup);
1235
1236 // Add registers spilled in libcall as liveins.
1237 for (auto &CS : CSI)
1238 MBB.addLiveIn(CS.getReg());
1239 }
1240
1241 // Manually spill values not spilled by libcall.
1242 const auto &NonLibcallCSI = getNonLibcallCSI(*MF, CSI);
1243 for (auto &CS : NonLibcallCSI) {
1244 // Insert the spill to the stack frame.
1245 Register Reg = CS.getReg();
1246 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1247 TII.storeRegToStackSlot(MBB, MI, Reg, !MBB.isLiveIn(Reg), CS.getFrameIdx(),
1248 RC, TRI, Register());
1249 }
1250
1251 return true;
1252 }
1253
restoreCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,MutableArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const1254 bool RISCVFrameLowering::restoreCalleeSavedRegisters(
1255 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1256 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
1257 if (CSI.empty())
1258 return true;
1259
1260 MachineFunction *MF = MBB.getParent();
1261 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1262 DebugLoc DL;
1263 if (MI != MBB.end() && !MI->isDebugInstr())
1264 DL = MI->getDebugLoc();
1265
1266 // Manually restore values not restored by libcall.
1267 // Keep the same order as in the prologue. There is no need to reverse the
1268 // order in the epilogue. In addition, the return address will be restored
1269 // first in the epilogue. It increases the opportunity to avoid the
1270 // load-to-use data hazard between loading RA and return by RA.
1271 // loadRegFromStackSlot can insert multiple instructions.
1272 const auto &NonLibcallCSI = getNonLibcallCSI(*MF, CSI);
1273 for (auto &CS : NonLibcallCSI) {
1274 Register Reg = CS.getReg();
1275 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1276 TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, TRI,
1277 Register());
1278 assert(MI != MBB.begin() && "loadRegFromStackSlot didn't insert any code!");
1279 }
1280
1281 const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI);
1282 if (RestoreLibCall) {
1283 // Add restore libcall via tail call.
1284 MachineBasicBlock::iterator NewMI =
1285 BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL))
1286 .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL)
1287 .setMIFlag(MachineInstr::FrameDestroy);
1288
1289 // Remove trailing returns, since the terminator is now a tail call to the
1290 // restore function.
1291 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
1292 NewMI->copyImplicitOps(*MF, *MI);
1293 MI->eraseFromParent();
1294 }
1295 }
1296
1297 return true;
1298 }
1299
enableShrinkWrapping(const MachineFunction & MF) const1300 bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
1301 // Keep the conventional code flow when not optimizing.
1302 if (MF.getFunction().hasOptNone())
1303 return false;
1304
1305 return true;
1306 }
1307
canUseAsPrologue(const MachineBasicBlock & MBB) const1308 bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
1309 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1310 const MachineFunction *MF = MBB.getParent();
1311 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1312
1313 if (!RVFI->useSaveRestoreLibCalls(*MF))
1314 return true;
1315
1316 // Inserting a call to a __riscv_save libcall requires the use of the register
1317 // t0 (X5) to hold the return address. Therefore if this register is already
1318 // used we can't insert the call.
1319
1320 RegScavenger RS;
1321 RS.enterBasicBlock(*TmpMBB);
1322 return !RS.isRegUsed(RISCV::X5);
1323 }
1324
canUseAsEpilogue(const MachineBasicBlock & MBB) const1325 bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
1326 const MachineFunction *MF = MBB.getParent();
1327 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
1328 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1329
1330 if (!RVFI->useSaveRestoreLibCalls(*MF))
1331 return true;
1332
1333 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
1334 // This means if we still need to continue executing code within this function
1335 // the restore cannot take place in this basic block.
1336
1337 if (MBB.succ_size() > 1)
1338 return false;
1339
1340 MachineBasicBlock *SuccMBB =
1341 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
1342
1343 // Doing a tail call should be safe if there are no successors, because either
1344 // we have a returning block or the end of the block is unreachable, so the
1345 // restore will be eliminated regardless.
1346 if (!SuccMBB)
1347 return true;
1348
1349 // The successor can only contain a return, since we would effectively be
1350 // replacing the successor with our own tail return at the end of our block.
1351 return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
1352 }
1353
isSupportedStackID(TargetStackID::Value ID) const1354 bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
1355 switch (ID) {
1356 case TargetStackID::Default:
1357 case TargetStackID::ScalableVector:
1358 return true;
1359 case TargetStackID::NoAlloc:
1360 case TargetStackID::SGPRSpill:
1361 case TargetStackID::WasmLocal:
1362 return false;
1363 }
1364 llvm_unreachable("Invalid TargetStackID::Value");
1365 }
1366
getStackIDForScalableVectors() const1367 TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const {
1368 return TargetStackID::ScalableVector;
1369 }
1370