xref: /openbsd-src/gnu/llvm/llvm/lib/Target/X86/X86FrameLowering.cpp (revision a96b36398fcfb4953e8190127da8bf074c7552f1)
1 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the X86 implementation of TargetFrameLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "X86FrameLowering.h"
14 #include "MCTargetDesc/X86MCTargetDesc.h"
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86ReturnProtectorLowering.h"
19 #include "X86Subtarget.h"
20 #include "X86TargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/EHPersonalities.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineFrameInfo.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineModuleInfo.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/WinEHFuncInfo.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Function.h"
33 #include "llvm/MC/MCAsmInfo.h"
34 #include "llvm/MC/MCObjectFileInfo.h"
35 #include "llvm/MC/MCSymbol.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Target/TargetOptions.h"
38 #include <cstdlib>
39 
40 #define DEBUG_TYPE "x86-fl"
41 
42 STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
43 STATISTIC(NumFrameExtraProbe,
44           "Number of extra stack probes generated in prologue");
45 
46 using namespace llvm;
47 
X86FrameLowering(const X86Subtarget & STI,MaybeAlign StackAlignOverride)48 X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
49                                    MaybeAlign StackAlignOverride)
50     : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(),
51                           STI.is64Bit() ? -8 : -4),
52       STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()), RPL() {
53   // Cache a bunch of frame-related predicates for this subtarget.
54   SlotSize = TRI->getSlotSize();
55   Is64Bit = STI.is64Bit();
56   IsLP64 = STI.isTarget64BitLP64();
57   // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
58   Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64();
59   StackPtr = TRI->getStackRegister();
60   SaveArgs = Is64Bit ? STI.getSaveArgs() : 0;
61 }
62 
hasReservedCallFrame(const MachineFunction & MF) const63 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
64   return !MF.getFrameInfo().hasVarSizedObjects() &&
65          !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() &&
66          !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall();
67 }
68 
69 /// canSimplifyCallFramePseudos - If there is a reserved call frame, the
70 /// call frame pseudos can be simplified.  Having a FP, as in the default
71 /// implementation, is not sufficient here since we can't always use it.
72 /// Use a more nuanced condition.
73 bool
canSimplifyCallFramePseudos(const MachineFunction & MF) const74 X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
75   return hasReservedCallFrame(MF) ||
76          MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
77          (hasFP(MF) && !TRI->hasStackRealignment(MF)) ||
78          TRI->hasBasePointer(MF);
79 }
80 
81 // needsFrameIndexResolution - Do we need to perform FI resolution for
82 // this function. Normally, this is required only when the function
83 // has any stack objects. However, FI resolution actually has another job,
84 // not apparent from the title - it resolves callframesetup/destroy
85 // that were not simplified earlier.
86 // So, this is required for x86 functions that have push sequences even
87 // when there are no stack objects.
88 bool
needsFrameIndexResolution(const MachineFunction & MF) const89 X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const {
90   return MF.getFrameInfo().hasStackObjects() ||
91          MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
92 }
93 
94 /// hasFP - Return true if the specified function should have a dedicated frame
95 /// pointer register.  This is true if the function has variable sized allocas
96 /// or if frame pointer elimination is disabled.
hasFP(const MachineFunction & MF) const97 bool X86FrameLowering::hasFP(const MachineFunction &MF) const {
98   const MachineFrameInfo &MFI = MF.getFrameInfo();
99   return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
100           TRI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
101           MFI.isFrameAddressTaken() || MFI.hasOpaqueSPAdjustment() ||
102           MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
103           MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
104           MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() ||
105           MFI.hasStackMap() || MFI.hasPatchPoint() ||
106           (isWin64Prologue(MF) && MFI.hasCopyImplyingStackAdjustment()) ||
107           SaveArgs);
108 }
109 
getSUBriOpcode(bool IsLP64,int64_t Imm)110 static unsigned getSUBriOpcode(bool IsLP64, int64_t Imm) {
111   if (IsLP64) {
112     if (isInt<8>(Imm))
113       return X86::SUB64ri8;
114     return X86::SUB64ri32;
115   } else {
116     if (isInt<8>(Imm))
117       return X86::SUB32ri8;
118     return X86::SUB32ri;
119   }
120 }
121 
getADDriOpcode(bool IsLP64,int64_t Imm)122 static unsigned getADDriOpcode(bool IsLP64, int64_t Imm) {
123   if (IsLP64) {
124     if (isInt<8>(Imm))
125       return X86::ADD64ri8;
126     return X86::ADD64ri32;
127   } else {
128     if (isInt<8>(Imm))
129       return X86::ADD32ri8;
130     return X86::ADD32ri;
131   }
132 }
133 
getSUBrrOpcode(bool IsLP64)134 static unsigned getSUBrrOpcode(bool IsLP64) {
135   return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
136 }
137 
getADDrrOpcode(bool IsLP64)138 static unsigned getADDrrOpcode(bool IsLP64) {
139   return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
140 }
141 
getANDriOpcode(bool IsLP64,int64_t Imm)142 static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
143   if (IsLP64) {
144     if (isInt<8>(Imm))
145       return X86::AND64ri8;
146     return X86::AND64ri32;
147   }
148   if (isInt<8>(Imm))
149     return X86::AND32ri8;
150   return X86::AND32ri;
151 }
152 
getLEArOpcode(bool IsLP64)153 static unsigned getLEArOpcode(bool IsLP64) {
154   return IsLP64 ? X86::LEA64r : X86::LEA32r;
155 }
156 
getMOVriOpcode(bool Use64BitReg,int64_t Imm)157 static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm) {
158   if (Use64BitReg) {
159     if (isUInt<32>(Imm))
160       return X86::MOV32ri64;
161     if (isInt<32>(Imm))
162       return X86::MOV64ri32;
163     return X86::MOV64ri;
164   }
165   return X86::MOV32ri;
166 }
167 
isEAXLiveIn(MachineBasicBlock & MBB)168 static bool isEAXLiveIn(MachineBasicBlock &MBB) {
169   for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) {
170     unsigned Reg = RegMask.PhysReg;
171 
172     if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
173         Reg == X86::AH || Reg == X86::AL)
174       return true;
175   }
176 
177   return false;
178 }
179 
180 /// Check if the flags need to be preserved before the terminators.
181 /// This would be the case, if the eflags is live-in of the region
182 /// composed by the terminators or live-out of that region, without
183 /// being defined by a terminator.
184 static bool
flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock & MBB)185 flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
186   for (const MachineInstr &MI : MBB.terminators()) {
187     bool BreakNext = false;
188     for (const MachineOperand &MO : MI.operands()) {
189       if (!MO.isReg())
190         continue;
191       Register Reg = MO.getReg();
192       if (Reg != X86::EFLAGS)
193         continue;
194 
195       // This terminator needs an eflags that is not defined
196       // by a previous another terminator:
197       // EFLAGS is live-in of the region composed by the terminators.
198       if (!MO.isDef())
199         return true;
200       // This terminator defines the eflags, i.e., we don't need to preserve it.
201       // However, we still need to check this specific terminator does not
202       // read a live-in value.
203       BreakNext = true;
204     }
205     // We found a definition of the eflags, no need to preserve them.
206     if (BreakNext)
207       return false;
208   }
209 
210   // None of the terminators use or define the eflags.
211   // Check if they are live-out, that would imply we need to preserve them.
212   for (const MachineBasicBlock *Succ : MBB.successors())
213     if (Succ->isLiveIn(X86::EFLAGS))
214       return true;
215 
216   return false;
217 }
218 
219 /// emitSPUpdate - Emit a series of instructions to increment / decrement the
220 /// stack pointer by a constant value.
emitSPUpdate(MachineBasicBlock & MBB,MachineBasicBlock::iterator & MBBI,const DebugLoc & DL,int64_t NumBytes,bool InEpilogue) const221 void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
222                                     MachineBasicBlock::iterator &MBBI,
223                                     const DebugLoc &DL,
224                                     int64_t NumBytes, bool InEpilogue) const {
225   bool isSub = NumBytes < 0;
226   uint64_t Offset = isSub ? -NumBytes : NumBytes;
227   MachineInstr::MIFlag Flag =
228       isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy;
229 
230   uint64_t Chunk = (1LL << 31) - 1;
231 
232   MachineFunction &MF = *MBB.getParent();
233   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
234   const X86TargetLowering &TLI = *STI.getTargetLowering();
235   const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
236 
237   // It's ok to not take into account large chunks when probing, as the
238   // allocation is split in smaller chunks anyway.
239   if (EmitInlineStackProbe && !InEpilogue) {
240 
241     // This pseudo-instruction is going to be expanded, potentially using a
242     // loop, by inlineStackProbe().
243     BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING)).addImm(Offset);
244     return;
245   } else if (Offset > Chunk) {
246     // Rather than emit a long series of instructions for large offsets,
247     // load the offset into a register and do one sub/add
248     unsigned Reg = 0;
249     unsigned Rax = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
250 
251     if (isSub && !isEAXLiveIn(MBB))
252       Reg = Rax;
253     else
254       Reg = TRI->findDeadCallerSavedReg(MBB, MBBI);
255 
256     unsigned AddSubRROpc =
257         isSub ? getSUBrrOpcode(Is64Bit) : getADDrrOpcode(Is64Bit);
258     if (Reg) {
259       BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Offset)), Reg)
260           .addImm(Offset)
261           .setMIFlag(Flag);
262       MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AddSubRROpc), StackPtr)
263                              .addReg(StackPtr)
264                              .addReg(Reg);
265       MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
266       return;
267     } else if (Offset > 8 * Chunk) {
268       // If we would need more than 8 add or sub instructions (a >16GB stack
269       // frame), it's worth spilling RAX to materialize this immediate.
270       //   pushq %rax
271       //   movabsq +-$Offset+-SlotSize, %rax
272       //   addq %rsp, %rax
273       //   xchg %rax, (%rsp)
274       //   movq (%rsp), %rsp
275       assert(Is64Bit && "can't have 32-bit 16GB stack frame");
276       BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
277           .addReg(Rax, RegState::Kill)
278           .setMIFlag(Flag);
279       // Subtract is not commutative, so negate the offset and always use add.
280       // Subtract 8 less and add 8 more to account for the PUSH we just did.
281       if (isSub)
282         Offset = -(Offset - SlotSize);
283       else
284         Offset = Offset + SlotSize;
285       BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Offset)), Rax)
286           .addImm(Offset)
287           .setMIFlag(Flag);
288       MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(X86::ADD64rr), Rax)
289                              .addReg(Rax)
290                              .addReg(StackPtr);
291       MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
292       // Exchange the new SP in RAX with the top of the stack.
293       addRegOffset(
294           BuildMI(MBB, MBBI, DL, TII.get(X86::XCHG64rm), Rax).addReg(Rax),
295           StackPtr, false, 0);
296       // Load new SP from the top of the stack into RSP.
297       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), StackPtr),
298                    StackPtr, false, 0);
299       return;
300     }
301   }
302 
303   while (Offset) {
304     uint64_t ThisVal = std::min(Offset, Chunk);
305     if (ThisVal == SlotSize) {
306       // Use push / pop for slot sized adjustments as a size optimization. We
307       // need to find a dead register when using pop.
308       unsigned Reg = isSub
309         ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
310         : TRI->findDeadCallerSavedReg(MBB, MBBI);
311       if (Reg) {
312         unsigned Opc = isSub
313           ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
314           : (Is64Bit ? X86::POP64r  : X86::POP32r);
315         BuildMI(MBB, MBBI, DL, TII.get(Opc))
316             .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub))
317             .setMIFlag(Flag);
318         Offset -= ThisVal;
319         continue;
320       }
321     }
322 
323     BuildStackAdjustment(MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue)
324         .setMIFlag(Flag);
325 
326     Offset -= ThisVal;
327   }
328 }
329 
BuildStackAdjustment(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,int64_t Offset,bool InEpilogue) const330 MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
331     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
332     const DebugLoc &DL, int64_t Offset, bool InEpilogue) const {
333   assert(Offset != 0 && "zero offset stack adjustment requested");
334 
335   // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
336   // is tricky.
337   bool UseLEA;
338   if (!InEpilogue) {
339     // Check if inserting the prologue at the beginning
340     // of MBB would require to use LEA operations.
341     // We need to use LEA operations if EFLAGS is live in, because
342     // it means an instruction will read it before it gets defined.
343     UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS);
344   } else {
345     // If we can use LEA for SP but we shouldn't, check that none
346     // of the terminators uses the eflags. Otherwise we will insert
347     // a ADD that will redefine the eflags and break the condition.
348     // Alternatively, we could move the ADD, but this may not be possible
349     // and is an optimization anyway.
350     UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent());
351     if (UseLEA && !STI.useLeaForSP())
352       UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB);
353     // If that assert breaks, that means we do not do the right thing
354     // in canUseAsEpilogue.
355     assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) &&
356            "We shouldn't have allowed this insertion point");
357   }
358 
359   MachineInstrBuilder MI;
360   if (UseLEA) {
361     MI = addRegOffset(BuildMI(MBB, MBBI, DL,
362                               TII.get(getLEArOpcode(Uses64BitFramePtr)),
363                               StackPtr),
364                       StackPtr, false, Offset);
365   } else {
366     bool IsSub = Offset < 0;
367     uint64_t AbsOffset = IsSub ? -Offset : Offset;
368     const unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset)
369                                : getADDriOpcode(Uses64BitFramePtr, AbsOffset);
370     MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
371              .addReg(StackPtr)
372              .addImm(AbsOffset);
373     MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
374   }
375   return MI;
376 }
377 
mergeSPUpdates(MachineBasicBlock & MBB,MachineBasicBlock::iterator & MBBI,bool doMergeWithPrevious) const378 int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
379                                      MachineBasicBlock::iterator &MBBI,
380                                      bool doMergeWithPrevious) const {
381   if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
382       (!doMergeWithPrevious && MBBI == MBB.end()))
383     return 0;
384 
385   MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI;
386 
387   PI = skipDebugInstructionsBackward(PI, MBB.begin());
388   // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI
389   // instruction, and that there are no DBG_VALUE or other instructions between
390   // ADD/SUB/LEA and its corresponding CFI instruction.
391   /* TODO: Add support for the case where there are multiple CFI instructions
392     below the ADD/SUB/LEA, e.g.:
393     ...
394     add
395     cfi_def_cfa_offset
396     cfi_offset
397     ...
398   */
399   if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
400     PI = std::prev(PI);
401 
402   unsigned Opc = PI->getOpcode();
403   int Offset = 0;
404 
405   if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
406        Opc == X86::ADD32ri || Opc == X86::ADD32ri8) &&
407       PI->getOperand(0).getReg() == StackPtr){
408     assert(PI->getOperand(1).getReg() == StackPtr);
409     Offset = PI->getOperand(2).getImm();
410   } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
411              PI->getOperand(0).getReg() == StackPtr &&
412              PI->getOperand(1).getReg() == StackPtr &&
413              PI->getOperand(2).getImm() == 1 &&
414              PI->getOperand(3).getReg() == X86::NoRegister &&
415              PI->getOperand(5).getReg() == X86::NoRegister) {
416     // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
417     Offset = PI->getOperand(4).getImm();
418   } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
419               Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
420              PI->getOperand(0).getReg() == StackPtr) {
421     assert(PI->getOperand(1).getReg() == StackPtr);
422     Offset = -PI->getOperand(2).getImm();
423   } else
424     return 0;
425 
426   PI = MBB.erase(PI);
427   if (PI != MBB.end() && PI->isCFIInstruction()) {
428     auto CIs = MBB.getParent()->getFrameInstructions();
429     MCCFIInstruction CI = CIs[PI->getOperand(0).getCFIIndex()];
430     if (CI.getOperation() == MCCFIInstruction::OpDefCfaOffset ||
431         CI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
432       PI = MBB.erase(PI);
433   }
434   if (!doMergeWithPrevious)
435     MBBI = skipDebugInstructionsForward(PI, MBB.end());
436 
437   return Offset;
438 }
439 
BuildCFI(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,const MCCFIInstruction & CFIInst,MachineInstr::MIFlag Flag) const440 void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
441                                 MachineBasicBlock::iterator MBBI,
442                                 const DebugLoc &DL,
443                                 const MCCFIInstruction &CFIInst,
444                                 MachineInstr::MIFlag Flag) const {
445   MachineFunction &MF = *MBB.getParent();
446   unsigned CFIIndex = MF.addFrameInst(CFIInst);
447   BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
448       .addCFIIndex(CFIIndex)
449       .setMIFlag(Flag);
450 }
451 
452 /// Emits Dwarf Info specifying offsets of callee saved registers and
453 /// frame pointer. This is called only when basic block sections are enabled.
emitCalleeSavedFrameMovesFullCFA(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const454 void X86FrameLowering::emitCalleeSavedFrameMovesFullCFA(
455     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
456   MachineFunction &MF = *MBB.getParent();
457   if (!hasFP(MF)) {
458     emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true);
459     return;
460   }
461   const MachineModuleInfo &MMI = MF.getMMI();
462   const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
463   const Register FramePtr = TRI->getFrameRegister(MF);
464   const Register MachineFramePtr =
465       STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(FramePtr, 64))
466                                : FramePtr;
467   unsigned DwarfReg = MRI->getDwarfRegNum(MachineFramePtr, true);
468   // Offset = space for return address + size of the frame pointer itself.
469   unsigned Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4);
470   BuildCFI(MBB, MBBI, DebugLoc{},
471            MCCFIInstruction::createOffset(nullptr, DwarfReg, -Offset));
472   emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true);
473 }
474 
emitCalleeSavedFrameMoves(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool IsPrologue) const475 void X86FrameLowering::emitCalleeSavedFrameMoves(
476     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
477     const DebugLoc &DL, bool IsPrologue) const {
478   MachineFunction &MF = *MBB.getParent();
479   MachineFrameInfo &MFI = MF.getFrameInfo();
480   MachineModuleInfo &MMI = MF.getMMI();
481   const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
482 
483   // Add callee saved registers to move list.
484   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
485 
486   // Calculate offsets.
487   for (const CalleeSavedInfo &I : CSI) {
488     int64_t Offset = MFI.getObjectOffset(I.getFrameIdx());
489     Register Reg = I.getReg();
490     unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
491 
492     if (IsPrologue) {
493       BuildCFI(MBB, MBBI, DL,
494                MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
495     } else {
496       BuildCFI(MBB, MBBI, DL,
497                MCCFIInstruction::createRestore(nullptr, DwarfReg));
498     }
499   }
500 }
501 
emitZeroCallUsedRegs(BitVector RegsToZero,MachineBasicBlock & MBB) const502 void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
503                                             MachineBasicBlock &MBB) const {
504   const MachineFunction &MF = *MBB.getParent();
505 
506   // Insertion point.
507   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
508 
509   // Fake a debug loc.
510   DebugLoc DL;
511   if (MBBI != MBB.end())
512     DL = MBBI->getDebugLoc();
513 
514   // Zero out FP stack if referenced. Do this outside of the loop below so that
515   // it's done only once.
516   const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
517   for (MCRegister Reg : RegsToZero.set_bits()) {
518     if (!X86::RFP80RegClass.contains(Reg))
519       continue;
520 
521     unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
522     for (unsigned i = 0; i != NumFPRegs; ++i)
523       BuildMI(MBB, MBBI, DL, TII.get(X86::LD_F0));
524 
525     for (unsigned i = 0; i != NumFPRegs; ++i)
526       BuildMI(MBB, MBBI, DL, TII.get(X86::ST_FPrr)).addReg(X86::ST0);
527     break;
528   }
529 
530   // For GPRs, we only care to clear out the 32-bit register.
531   BitVector GPRsToZero(TRI->getNumRegs());
532   for (MCRegister Reg : RegsToZero.set_bits())
533     if (TRI->isGeneralPurposeRegister(MF, Reg)) {
534       GPRsToZero.set(getX86SubSuperRegisterOrZero(Reg, 32));
535       RegsToZero.reset(Reg);
536     }
537 
538   for (MCRegister Reg : GPRsToZero.set_bits())
539     BuildMI(MBB, MBBI, DL, TII.get(X86::XOR32rr), Reg)
540         .addReg(Reg, RegState::Undef)
541         .addReg(Reg, RegState::Undef);
542 
543   // Zero out registers.
544   for (MCRegister Reg : RegsToZero.set_bits()) {
545     if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
546       // FIXME: Ignore MMX registers?
547       continue;
548 
549     unsigned XorOp;
550     if (X86::VR128RegClass.contains(Reg)) {
551       // XMM#
552       if (!ST.hasSSE1())
553         continue;
554       XorOp = X86::PXORrr;
555     } else if (X86::VR256RegClass.contains(Reg)) {
556       // YMM#
557       if (!ST.hasAVX())
558         continue;
559       XorOp = X86::VPXORrr;
560     } else if (X86::VR512RegClass.contains(Reg)) {
561       // ZMM#
562       if (!ST.hasAVX512())
563         continue;
564       XorOp = X86::VPXORYrr;
565     } else if (X86::VK1RegClass.contains(Reg) ||
566                X86::VK2RegClass.contains(Reg) ||
567                X86::VK4RegClass.contains(Reg) ||
568                X86::VK8RegClass.contains(Reg) ||
569                X86::VK16RegClass.contains(Reg)) {
570       if (!ST.hasVLX())
571         continue;
572       XorOp = ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
573     } else {
574       continue;
575     }
576 
577     BuildMI(MBB, MBBI, DL, TII.get(XorOp), Reg)
578       .addReg(Reg, RegState::Undef)
579       .addReg(Reg, RegState::Undef);
580   }
581 }
582 
emitStackProbe(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog,std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const583 void X86FrameLowering::emitStackProbe(
584     MachineFunction &MF, MachineBasicBlock &MBB,
585     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
586     std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
587   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
588   if (STI.isTargetWindowsCoreCLR()) {
589     if (InProlog) {
590       BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING))
591           .addImm(0 /* no explicit stack size */);
592     } else {
593       emitStackProbeInline(MF, MBB, MBBI, DL, false);
594     }
595   } else {
596     emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum);
597   }
598 }
599 
stackProbeFunctionModifiesSP() const600 bool X86FrameLowering::stackProbeFunctionModifiesSP() const {
601   return STI.isOSWindows() && !STI.isTargetWin64();
602 }
603 
inlineStackProbe(MachineFunction & MF,MachineBasicBlock & PrologMBB) const604 void X86FrameLowering::inlineStackProbe(MachineFunction &MF,
605                                         MachineBasicBlock &PrologMBB) const {
606   auto Where = llvm::find_if(PrologMBB, [](MachineInstr &MI) {
607     return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
608   });
609   if (Where != PrologMBB.end()) {
610     DebugLoc DL = PrologMBB.findDebugLoc(Where);
611     emitStackProbeInline(MF, PrologMBB, Where, DL, true);
612     Where->eraseFromParent();
613   }
614 }
615 
emitStackProbeInline(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog) const616 void X86FrameLowering::emitStackProbeInline(MachineFunction &MF,
617                                             MachineBasicBlock &MBB,
618                                             MachineBasicBlock::iterator MBBI,
619                                             const DebugLoc &DL,
620                                             bool InProlog) const {
621   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
622   if (STI.isTargetWindowsCoreCLR() && STI.is64Bit())
623     emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog);
624   else
625     emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog);
626 }
627 
emitStackProbeInlineGeneric(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog) const628 void X86FrameLowering::emitStackProbeInlineGeneric(
629     MachineFunction &MF, MachineBasicBlock &MBB,
630     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
631   MachineInstr &AllocWithProbe = *MBBI;
632   uint64_t Offset = AllocWithProbe.getOperand(0).getImm();
633 
634   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
635   const X86TargetLowering &TLI = *STI.getTargetLowering();
636   assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) &&
637          "different expansion expected for CoreCLR 64 bit");
638 
639   const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
640   uint64_t ProbeChunk = StackProbeSize * 8;
641 
642   uint64_t MaxAlign =
643       TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
644 
645   // Synthesize a loop or unroll it, depending on the number of iterations.
646   // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left
647   // between the unaligned rsp and current rsp.
648   if (Offset > ProbeChunk) {
649     emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset,
650                                     MaxAlign % StackProbeSize);
651   } else {
652     emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset,
653                                      MaxAlign % StackProbeSize);
654   }
655 }
656 
emitStackProbeInlineGenericBlock(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,uint64_t Offset,uint64_t AlignOffset) const657 void X86FrameLowering::emitStackProbeInlineGenericBlock(
658     MachineFunction &MF, MachineBasicBlock &MBB,
659     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
660     uint64_t AlignOffset) const {
661 
662   const bool NeedsDwarfCFI = needsDwarfCFI(MF);
663   const bool HasFP = hasFP(MF);
664   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
665   const X86TargetLowering &TLI = *STI.getTargetLowering();
666   const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
667   const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
668 
669   uint64_t CurrentOffset = 0;
670 
671   assert(AlignOffset < StackProbeSize);
672 
673   // If the offset is so small it fits within a page, there's nothing to do.
674   if (StackProbeSize < Offset + AlignOffset) {
675 
676     uint64_t StackAdjustment = StackProbeSize - AlignOffset;
677     BuildStackAdjustment(MBB, MBBI, DL, -StackAdjustment, /*InEpilogue=*/false)
678         .setMIFlag(MachineInstr::FrameSetup);
679     if (!HasFP && NeedsDwarfCFI) {
680       BuildCFI(
681           MBB, MBBI, DL,
682           MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment));
683     }
684 
685     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
686                      .setMIFlag(MachineInstr::FrameSetup),
687                  StackPtr, false, 0)
688         .addImm(0)
689         .setMIFlag(MachineInstr::FrameSetup);
690     NumFrameExtraProbe++;
691     CurrentOffset = StackProbeSize - AlignOffset;
692   }
693 
694   // For the next N - 1 pages, just probe. I tried to take advantage of
695   // natural probes but it implies much more logic and there was very few
696   // interesting natural probes to interleave.
697   while (CurrentOffset + StackProbeSize < Offset) {
698     BuildStackAdjustment(MBB, MBBI, DL, -StackProbeSize, /*InEpilogue=*/false)
699         .setMIFlag(MachineInstr::FrameSetup);
700 
701     if (!HasFP && NeedsDwarfCFI) {
702       BuildCFI(
703           MBB, MBBI, DL,
704           MCCFIInstruction::createAdjustCfaOffset(nullptr, StackProbeSize));
705     }
706     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
707                      .setMIFlag(MachineInstr::FrameSetup),
708                  StackPtr, false, 0)
709         .addImm(0)
710         .setMIFlag(MachineInstr::FrameSetup);
711     NumFrameExtraProbe++;
712     CurrentOffset += StackProbeSize;
713   }
714 
715   // No need to probe the tail, it is smaller than a Page.
716   uint64_t ChunkSize = Offset - CurrentOffset;
717   if (ChunkSize == SlotSize) {
718     // Use push for slot sized adjustments as a size optimization,
719     // like emitSPUpdate does when not probing.
720     unsigned Reg = Is64Bit ? X86::RAX : X86::EAX;
721     unsigned Opc = Is64Bit ? X86::PUSH64r : X86::PUSH32r;
722     BuildMI(MBB, MBBI, DL, TII.get(Opc))
723         .addReg(Reg, RegState::Undef)
724         .setMIFlag(MachineInstr::FrameSetup);
725   } else {
726     BuildStackAdjustment(MBB, MBBI, DL, -ChunkSize, /*InEpilogue=*/false)
727         .setMIFlag(MachineInstr::FrameSetup);
728   }
729   // No need to adjust Dwarf CFA offset here, the last position of the stack has
730   // been defined
731 }
732 
emitStackProbeInlineGenericLoop(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,uint64_t Offset,uint64_t AlignOffset) const733 void X86FrameLowering::emitStackProbeInlineGenericLoop(
734     MachineFunction &MF, MachineBasicBlock &MBB,
735     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
736     uint64_t AlignOffset) const {
737   assert(Offset && "null offset");
738 
739   assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
740              MachineBasicBlock::LQR_Live &&
741          "Inline stack probe loop will clobber live EFLAGS.");
742 
743   const bool NeedsDwarfCFI = needsDwarfCFI(MF);
744   const bool HasFP = hasFP(MF);
745   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
746   const X86TargetLowering &TLI = *STI.getTargetLowering();
747   const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
748   const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
749 
750   if (AlignOffset) {
751     if (AlignOffset < StackProbeSize) {
752       // Perform a first smaller allocation followed by a probe.
753       BuildStackAdjustment(MBB, MBBI, DL, -AlignOffset, /*InEpilogue=*/false)
754           .setMIFlag(MachineInstr::FrameSetup);
755 
756       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
757                        .setMIFlag(MachineInstr::FrameSetup),
758                    StackPtr, false, 0)
759           .addImm(0)
760           .setMIFlag(MachineInstr::FrameSetup);
761       NumFrameExtraProbe++;
762       Offset -= AlignOffset;
763     }
764   }
765 
766   // Synthesize a loop
767   NumFrameLoopProbe++;
768   const BasicBlock *LLVM_BB = MBB.getBasicBlock();
769 
770   MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(LLVM_BB);
771   MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(LLVM_BB);
772 
773   MachineFunction::iterator MBBIter = ++MBB.getIterator();
774   MF.insert(MBBIter, testMBB);
775   MF.insert(MBBIter, tailMBB);
776 
777   Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
778                               : Is64Bit         ? X86::R11D
779                                                 : X86::EAX;
780 
781   BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
782       .addReg(StackPtr)
783       .setMIFlag(MachineInstr::FrameSetup);
784 
785   // save loop bound
786   {
787     const unsigned BoundOffset = alignDown(Offset, StackProbeSize);
788     const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr, BoundOffset);
789     BuildMI(MBB, MBBI, DL, TII.get(SUBOpc), FinalStackProbed)
790         .addReg(FinalStackProbed)
791         .addImm(BoundOffset)
792         .setMIFlag(MachineInstr::FrameSetup);
793 
794     // while in the loop, use loop-invariant reg for CFI,
795     // instead of the stack pointer, which changes during the loop
796     if (!HasFP && NeedsDwarfCFI) {
797       // x32 uses the same DWARF register numbers as x86-64,
798       // so there isn't a register number for r11d, we must use r11 instead
799       const Register DwarfFinalStackProbed =
800           STI.isTarget64BitILP32()
801               ? Register(getX86SubSuperRegister(FinalStackProbed, 64))
802               : FinalStackProbed;
803 
804       BuildCFI(MBB, MBBI, DL,
805                MCCFIInstruction::createDefCfaRegister(
806                    nullptr, TRI->getDwarfRegNum(DwarfFinalStackProbed, true)));
807       BuildCFI(MBB, MBBI, DL,
808                MCCFIInstruction::createAdjustCfaOffset(nullptr, BoundOffset));
809     }
810   }
811 
812   // allocate a page
813   BuildStackAdjustment(*testMBB, testMBB->end(), DL, -StackProbeSize,
814                        /*InEpilogue=*/false)
815       .setMIFlag(MachineInstr::FrameSetup);
816 
817   // touch the page
818   addRegOffset(BuildMI(testMBB, DL, TII.get(MovMIOpc))
819                    .setMIFlag(MachineInstr::FrameSetup),
820                StackPtr, false, 0)
821       .addImm(0)
822       .setMIFlag(MachineInstr::FrameSetup);
823 
824   // cmp with stack pointer bound
825   BuildMI(testMBB, DL, TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
826       .addReg(StackPtr)
827       .addReg(FinalStackProbed)
828       .setMIFlag(MachineInstr::FrameSetup);
829 
830   // jump
831   BuildMI(testMBB, DL, TII.get(X86::JCC_1))
832       .addMBB(testMBB)
833       .addImm(X86::COND_NE)
834       .setMIFlag(MachineInstr::FrameSetup);
835   testMBB->addSuccessor(testMBB);
836   testMBB->addSuccessor(tailMBB);
837 
838   // BB management
839   tailMBB->splice(tailMBB->end(), &MBB, MBBI, MBB.end());
840   tailMBB->transferSuccessorsAndUpdatePHIs(&MBB);
841   MBB.addSuccessor(testMBB);
842 
843   // handle tail
844   const uint64_t TailOffset = Offset % StackProbeSize;
845   MachineBasicBlock::iterator TailMBBIter = tailMBB->begin();
846   if (TailOffset) {
847     BuildStackAdjustment(*tailMBB, TailMBBIter, DL, -TailOffset,
848                          /*InEpilogue=*/false)
849         .setMIFlag(MachineInstr::FrameSetup);
850   }
851 
852   // after the loop, switch back to stack pointer for CFI
853   if (!HasFP && NeedsDwarfCFI) {
854     // x32 uses the same DWARF register numbers as x86-64,
855     // so there isn't a register number for esp, we must use rsp instead
856     const Register DwarfStackPtr =
857         STI.isTarget64BitILP32()
858             ? Register(getX86SubSuperRegister(StackPtr, 64))
859             : Register(StackPtr);
860 
861     BuildCFI(*tailMBB, TailMBBIter, DL,
862              MCCFIInstruction::createDefCfaRegister(
863                  nullptr, TRI->getDwarfRegNum(DwarfStackPtr, true)));
864   }
865 
866   // Update Live In information
867   recomputeLiveIns(*testMBB);
868   recomputeLiveIns(*tailMBB);
869 }
870 
emitStackProbeInlineWindowsCoreCLR64(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog) const871 void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
872     MachineFunction &MF, MachineBasicBlock &MBB,
873     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
874   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
875   assert(STI.is64Bit() && "different expansion needed for 32 bit");
876   assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
877   const TargetInstrInfo &TII = *STI.getInstrInfo();
878   const BasicBlock *LLVM_BB = MBB.getBasicBlock();
879 
880   assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
881              MachineBasicBlock::LQR_Live &&
882          "Inline stack probe loop will clobber live EFLAGS.");
883 
884   // RAX contains the number of bytes of desired stack adjustment.
885   // The handling here assumes this value has already been updated so as to
886   // maintain stack alignment.
887   //
888   // We need to exit with RSP modified by this amount and execute suitable
889   // page touches to notify the OS that we're growing the stack responsibly.
890   // All stack probing must be done without modifying RSP.
891   //
892   // MBB:
893   //    SizeReg = RAX;
894   //    ZeroReg = 0
895   //    CopyReg = RSP
896   //    Flags, TestReg = CopyReg - SizeReg
897   //    FinalReg = !Flags.Ovf ? TestReg : ZeroReg
898   //    LimitReg = gs magic thread env access
899   //    if FinalReg >= LimitReg goto ContinueMBB
900   // RoundBB:
901   //    RoundReg = page address of FinalReg
902   // LoopMBB:
903   //    LoopReg = PHI(LimitReg,ProbeReg)
904   //    ProbeReg = LoopReg - PageSize
905   //    [ProbeReg] = 0
906   //    if (ProbeReg > RoundReg) goto LoopMBB
907   // ContinueMBB:
908   //    RSP = RSP - RAX
909   //    [rest of original MBB]
910 
911   // Set up the new basic blocks
912   MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB);
913   MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
914   MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB);
915 
916   MachineFunction::iterator MBBIter = std::next(MBB.getIterator());
917   MF.insert(MBBIter, RoundMBB);
918   MF.insert(MBBIter, LoopMBB);
919   MF.insert(MBBIter, ContinueMBB);
920 
921   // Split MBB and move the tail portion down to ContinueMBB.
922   MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI);
923   ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end());
924   ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB);
925 
926   // Some useful constants
927   const int64_t ThreadEnvironmentStackLimit = 0x10;
928   const int64_t PageSize = 0x1000;
929   const int64_t PageMask = ~(PageSize - 1);
930 
931   // Registers we need. For the normal case we use virtual
932   // registers. For the prolog expansion we use RAX, RCX and RDX.
933   MachineRegisterInfo &MRI = MF.getRegInfo();
934   const TargetRegisterClass *RegClass = &X86::GR64RegClass;
935   const Register SizeReg = InProlog ? X86::RAX
936                                     : MRI.createVirtualRegister(RegClass),
937                  ZeroReg = InProlog ? X86::RCX
938                                     : MRI.createVirtualRegister(RegClass),
939                  CopyReg = InProlog ? X86::RDX
940                                     : MRI.createVirtualRegister(RegClass),
941                  TestReg = InProlog ? X86::RDX
942                                     : MRI.createVirtualRegister(RegClass),
943                  FinalReg = InProlog ? X86::RDX
944                                      : MRI.createVirtualRegister(RegClass),
945                  RoundedReg = InProlog ? X86::RDX
946                                        : MRI.createVirtualRegister(RegClass),
947                  LimitReg = InProlog ? X86::RCX
948                                      : MRI.createVirtualRegister(RegClass),
949                  JoinReg = InProlog ? X86::RCX
950                                     : MRI.createVirtualRegister(RegClass),
951                  ProbeReg = InProlog ? X86::RCX
952                                      : MRI.createVirtualRegister(RegClass);
953 
954   // SP-relative offsets where we can save RCX and RDX.
955   int64_t RCXShadowSlot = 0;
956   int64_t RDXShadowSlot = 0;
957 
958   // If inlining in the prolog, save RCX and RDX.
959   if (InProlog) {
960     // Compute the offsets. We need to account for things already
961     // pushed onto the stack at this point: return address, frame
962     // pointer (if used), and callee saves.
963     X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
964     const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
965     const bool HasFP = hasFP(MF);
966 
967     // Check if we need to spill RCX and/or RDX.
968     // Here we assume that no earlier prologue instruction changes RCX and/or
969     // RDX, so checking the block live-ins is enough.
970     const bool IsRCXLiveIn = MBB.isLiveIn(X86::RCX);
971     const bool IsRDXLiveIn = MBB.isLiveIn(X86::RDX);
972     int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
973     // Assign the initial slot to both registers, then change RDX's slot if both
974     // need to be spilled.
975     if (IsRCXLiveIn)
976       RCXShadowSlot = InitSlot;
977     if (IsRDXLiveIn)
978       RDXShadowSlot = InitSlot;
979     if (IsRDXLiveIn && IsRCXLiveIn)
980       RDXShadowSlot += 8;
981     // Emit the saves if needed.
982     if (IsRCXLiveIn)
983       addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
984                    RCXShadowSlot)
985           .addReg(X86::RCX);
986     if (IsRDXLiveIn)
987       addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
988                    RDXShadowSlot)
989           .addReg(X86::RDX);
990   } else {
991     // Not in the prolog. Copy RAX to a virtual reg.
992     BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX);
993   }
994 
995   // Add code to MBB to check for overflow and set the new target stack pointer
996   // to zero if so.
997   BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg)
998       .addReg(ZeroReg, RegState::Undef)
999       .addReg(ZeroReg, RegState::Undef);
1000   BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP);
1001   BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg)
1002       .addReg(CopyReg)
1003       .addReg(SizeReg);
1004   BuildMI(&MBB, DL, TII.get(X86::CMOV64rr), FinalReg)
1005       .addReg(TestReg)
1006       .addReg(ZeroReg)
1007       .addImm(X86::COND_B);
1008 
1009   // FinalReg now holds final stack pointer value, or zero if
1010   // allocation would overflow. Compare against the current stack
1011   // limit from the thread environment block. Note this limit is the
1012   // lowest touched page on the stack, not the point at which the OS
1013   // will cause an overflow exception, so this is just an optimization
1014   // to avoid unnecessarily touching pages that are below the current
1015   // SP but already committed to the stack by the OS.
1016   BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg)
1017       .addReg(0)
1018       .addImm(1)
1019       .addReg(0)
1020       .addImm(ThreadEnvironmentStackLimit)
1021       .addReg(X86::GS);
1022   BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg);
1023   // Jump if the desired stack pointer is at or above the stack limit.
1024   BuildMI(&MBB, DL, TII.get(X86::JCC_1)).addMBB(ContinueMBB).addImm(X86::COND_AE);
1025 
1026   // Add code to roundMBB to round the final stack pointer to a page boundary.
1027   RoundMBB->addLiveIn(FinalReg);
1028   BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg)
1029       .addReg(FinalReg)
1030       .addImm(PageMask);
1031   BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB);
1032 
1033   // LimitReg now holds the current stack limit, RoundedReg page-rounded
1034   // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
1035   // and probe until we reach RoundedReg.
1036   if (!InProlog) {
1037     BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg)
1038         .addReg(LimitReg)
1039         .addMBB(RoundMBB)
1040         .addReg(ProbeReg)
1041         .addMBB(LoopMBB);
1042   }
1043 
1044   LoopMBB->addLiveIn(JoinReg);
1045   addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg,
1046                false, -PageSize);
1047 
1048   // Probe by storing a byte onto the stack.
1049   BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi))
1050       .addReg(ProbeReg)
1051       .addImm(1)
1052       .addReg(0)
1053       .addImm(0)
1054       .addReg(0)
1055       .addImm(0);
1056 
1057   LoopMBB->addLiveIn(RoundedReg);
1058   BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr))
1059       .addReg(RoundedReg)
1060       .addReg(ProbeReg);
1061   BuildMI(LoopMBB, DL, TII.get(X86::JCC_1)).addMBB(LoopMBB).addImm(X86::COND_NE);
1062 
1063   MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
1064 
1065   // If in prolog, restore RDX and RCX.
1066   if (InProlog) {
1067     if (RCXShadowSlot) // It means we spilled RCX in the prologue.
1068       addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL,
1069                            TII.get(X86::MOV64rm), X86::RCX),
1070                    X86::RSP, false, RCXShadowSlot);
1071     if (RDXShadowSlot) // It means we spilled RDX in the prologue.
1072       addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL,
1073                            TII.get(X86::MOV64rm), X86::RDX),
1074                    X86::RSP, false, RDXShadowSlot);
1075   }
1076 
1077   // Now that the probing is done, add code to continueMBB to update
1078   // the stack pointer for real.
1079   ContinueMBB->addLiveIn(SizeReg);
1080   BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP)
1081       .addReg(X86::RSP)
1082       .addReg(SizeReg);
1083 
1084   // Add the control flow edges we need.
1085   MBB.addSuccessor(ContinueMBB);
1086   MBB.addSuccessor(RoundMBB);
1087   RoundMBB->addSuccessor(LoopMBB);
1088   LoopMBB->addSuccessor(ContinueMBB);
1089   LoopMBB->addSuccessor(LoopMBB);
1090 
1091   // Mark all the instructions added to the prolog as frame setup.
1092   if (InProlog) {
1093     for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
1094       BeforeMBBI->setFlag(MachineInstr::FrameSetup);
1095     }
1096     for (MachineInstr &MI : *RoundMBB) {
1097       MI.setFlag(MachineInstr::FrameSetup);
1098     }
1099     for (MachineInstr &MI : *LoopMBB) {
1100       MI.setFlag(MachineInstr::FrameSetup);
1101     }
1102     for (MachineInstr &MI :
1103          llvm::make_range(ContinueMBB->begin(), ContinueMBBI)) {
1104       MI.setFlag(MachineInstr::FrameSetup);
1105     }
1106   }
1107 }
1108 
emitStackProbeCall(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog,std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const1109 void X86FrameLowering::emitStackProbeCall(
1110     MachineFunction &MF, MachineBasicBlock &MBB,
1111     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
1112     std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
1113   bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
1114 
1115   // FIXME: Add indirect thunk support and remove this.
1116   if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
1117     report_fatal_error("Emitting stack probe calls on 64-bit with the large "
1118                        "code model and indirect thunks not yet implemented.");
1119 
1120   assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1121              MachineBasicBlock::LQR_Live &&
1122          "Stack probe calls will clobber live EFLAGS.");
1123 
1124   unsigned CallOp;
1125   if (Is64Bit)
1126     CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1127   else
1128     CallOp = X86::CALLpcrel32;
1129 
1130   StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF);
1131 
1132   MachineInstrBuilder CI;
1133   MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI);
1134 
1135   // All current stack probes take AX and SP as input, clobber flags, and
1136   // preserve all registers. x86_64 probes leave RSP unmodified.
1137   if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
1138     // For the large code model, we have to call through a register. Use R11,
1139     // as it is scratch in all supported calling conventions.
1140     BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11)
1141         .addExternalSymbol(MF.createExternalSymbolName(Symbol));
1142     CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11);
1143   } else {
1144     CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp))
1145         .addExternalSymbol(MF.createExternalSymbolName(Symbol));
1146   }
1147 
1148   unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX;
1149   unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP;
1150   CI.addReg(AX, RegState::Implicit)
1151       .addReg(SP, RegState::Implicit)
1152       .addReg(AX, RegState::Define | RegState::Implicit)
1153       .addReg(SP, RegState::Define | RegState::Implicit)
1154       .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit);
1155 
1156   MachineInstr *ModInst = CI;
1157   if (STI.isTargetWin64() || !STI.isOSWindows()) {
1158     // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves.
1159     // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
1160     // themselves. They also does not clobber %rax so we can reuse it when
1161     // adjusting %rsp.
1162     // All other platforms do not specify a particular ABI for the stack probe
1163     // function, so we arbitrarily define it to not adjust %esp/%rsp itself.
1164     ModInst =
1165         BuildMI(MBB, MBBI, DL, TII.get(getSUBrrOpcode(Uses64BitFramePtr)), SP)
1166             .addReg(SP)
1167             .addReg(AX);
1168   }
1169 
1170   // DebugInfo variable locations -- if there's an instruction number for the
1171   // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that
1172   // modifies SP.
1173   if (InstrNum) {
1174     if (STI.isTargetWin64() || !STI.isOSWindows()) {
1175       // Label destination operand of the subtract.
1176       MF.makeDebugValueSubstitution(*InstrNum,
1177                                     {ModInst->getDebugInstrNum(), 0});
1178     } else {
1179       // Label the call. The operand number is the penultimate operand, zero
1180       // based.
1181       unsigned SPDefOperand = ModInst->getNumOperands() - 2;
1182       MF.makeDebugValueSubstitution(
1183           *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand});
1184     }
1185   }
1186 
1187   if (InProlog) {
1188     // Apply the frame setup flag to all inserted instrs.
1189     for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
1190       ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
1191   }
1192 }
1193 
calculateSetFPREG(uint64_t SPAdjust)1194 static unsigned calculateSetFPREG(uint64_t SPAdjust) {
1195   // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
1196   // and might require smaller successive adjustments.
1197   const uint64_t Win64MaxSEHOffset = 128;
1198   uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
1199   // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
1200   return SEHFrameOffset & -16;
1201 }
1202 
1203 // If we're forcing a stack realignment we can't rely on just the frame
1204 // info, we need to know the ABI stack alignment as well in case we
1205 // have a call out.  Otherwise just make sure we have some alignment - we'll
1206 // go with the minimum SlotSize.
calculateMaxStackAlign(const MachineFunction & MF) const1207 uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
1208   const MachineFrameInfo &MFI = MF.getFrameInfo();
1209   Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment.
1210   Align StackAlign = getStackAlign();
1211   if (MF.getFunction().hasFnAttribute("stackrealign")) {
1212     if (MFI.hasCalls())
1213       MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1214     else if (MaxAlign < SlotSize)
1215       MaxAlign = Align(SlotSize);
1216   }
1217   return MaxAlign.value();
1218 }
1219 
BuildStackAlignAND(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,unsigned Reg,uint64_t MaxAlign) const1220 void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
1221                                           MachineBasicBlock::iterator MBBI,
1222                                           const DebugLoc &DL, unsigned Reg,
1223                                           uint64_t MaxAlign) const {
1224   uint64_t Val = -MaxAlign;
1225   unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val);
1226 
1227   MachineFunction &MF = *MBB.getParent();
1228   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1229   const X86TargetLowering &TLI = *STI.getTargetLowering();
1230   const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1231   const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1232 
1233   // We want to make sure that (in worst case) less than StackProbeSize bytes
1234   // are not probed after the AND. This assumption is used in
1235   // emitStackProbeInlineGeneric.
1236   if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1237     {
1238       NumFrameLoopProbe++;
1239       MachineBasicBlock *entryMBB =
1240           MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1241       MachineBasicBlock *headMBB =
1242           MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1243       MachineBasicBlock *bodyMBB =
1244           MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1245       MachineBasicBlock *footMBB =
1246           MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1247 
1248       MachineFunction::iterator MBBIter = MBB.getIterator();
1249       MF.insert(MBBIter, entryMBB);
1250       MF.insert(MBBIter, headMBB);
1251       MF.insert(MBBIter, bodyMBB);
1252       MF.insert(MBBIter, footMBB);
1253       const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1254       Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
1255                                   : Is64Bit         ? X86::R11D
1256                                                     : X86::EAX;
1257 
1258       // Setup entry block
1259       {
1260 
1261         entryMBB->splice(entryMBB->end(), &MBB, MBB.begin(), MBBI);
1262         BuildMI(entryMBB, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
1263             .addReg(StackPtr)
1264             .setMIFlag(MachineInstr::FrameSetup);
1265         MachineInstr *MI =
1266             BuildMI(entryMBB, DL, TII.get(AndOp), FinalStackProbed)
1267                 .addReg(FinalStackProbed)
1268                 .addImm(Val)
1269                 .setMIFlag(MachineInstr::FrameSetup);
1270 
1271         // The EFLAGS implicit def is dead.
1272         MI->getOperand(3).setIsDead();
1273 
1274         BuildMI(entryMBB, DL,
1275                 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1276             .addReg(FinalStackProbed)
1277             .addReg(StackPtr)
1278             .setMIFlag(MachineInstr::FrameSetup);
1279         BuildMI(entryMBB, DL, TII.get(X86::JCC_1))
1280             .addMBB(&MBB)
1281             .addImm(X86::COND_E)
1282             .setMIFlag(MachineInstr::FrameSetup);
1283         entryMBB->addSuccessor(headMBB);
1284         entryMBB->addSuccessor(&MBB);
1285       }
1286 
1287       // Loop entry block
1288 
1289       {
1290         const unsigned SUBOpc =
1291             getSUBriOpcode(Uses64BitFramePtr, StackProbeSize);
1292         BuildMI(headMBB, DL, TII.get(SUBOpc), StackPtr)
1293             .addReg(StackPtr)
1294             .addImm(StackProbeSize)
1295             .setMIFlag(MachineInstr::FrameSetup);
1296 
1297         BuildMI(headMBB, DL,
1298                 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1299             .addReg(StackPtr)
1300             .addReg(FinalStackProbed)
1301             .setMIFlag(MachineInstr::FrameSetup);
1302 
1303         // jump to the footer if StackPtr < FinalStackProbed
1304         BuildMI(headMBB, DL, TII.get(X86::JCC_1))
1305             .addMBB(footMBB)
1306             .addImm(X86::COND_B)
1307             .setMIFlag(MachineInstr::FrameSetup);
1308 
1309         headMBB->addSuccessor(bodyMBB);
1310         headMBB->addSuccessor(footMBB);
1311       }
1312 
1313       // setup loop body
1314       {
1315         addRegOffset(BuildMI(bodyMBB, DL, TII.get(MovMIOpc))
1316                          .setMIFlag(MachineInstr::FrameSetup),
1317                      StackPtr, false, 0)
1318             .addImm(0)
1319             .setMIFlag(MachineInstr::FrameSetup);
1320 
1321         const unsigned SUBOpc =
1322             getSUBriOpcode(Uses64BitFramePtr, StackProbeSize);
1323         BuildMI(bodyMBB, DL, TII.get(SUBOpc), StackPtr)
1324             .addReg(StackPtr)
1325             .addImm(StackProbeSize)
1326             .setMIFlag(MachineInstr::FrameSetup);
1327 
1328         // cmp with stack pointer bound
1329         BuildMI(bodyMBB, DL,
1330                 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1331             .addReg(FinalStackProbed)
1332             .addReg(StackPtr)
1333             .setMIFlag(MachineInstr::FrameSetup);
1334 
1335         // jump back while FinalStackProbed < StackPtr
1336         BuildMI(bodyMBB, DL, TII.get(X86::JCC_1))
1337             .addMBB(bodyMBB)
1338             .addImm(X86::COND_B)
1339             .setMIFlag(MachineInstr::FrameSetup);
1340         bodyMBB->addSuccessor(bodyMBB);
1341         bodyMBB->addSuccessor(footMBB);
1342       }
1343 
1344       // setup loop footer
1345       {
1346         BuildMI(footMBB, DL, TII.get(TargetOpcode::COPY), StackPtr)
1347             .addReg(FinalStackProbed)
1348             .setMIFlag(MachineInstr::FrameSetup);
1349         addRegOffset(BuildMI(footMBB, DL, TII.get(MovMIOpc))
1350                          .setMIFlag(MachineInstr::FrameSetup),
1351                      StackPtr, false, 0)
1352             .addImm(0)
1353             .setMIFlag(MachineInstr::FrameSetup);
1354         footMBB->addSuccessor(&MBB);
1355       }
1356 
1357       recomputeLiveIns(*headMBB);
1358       recomputeLiveIns(*bodyMBB);
1359       recomputeLiveIns(*footMBB);
1360       recomputeLiveIns(MBB);
1361     }
1362   } else {
1363     MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg)
1364                            .addReg(Reg)
1365                            .addImm(Val)
1366                            .setMIFlag(MachineInstr::FrameSetup);
1367 
1368     // The EFLAGS implicit def is dead.
1369     MI->getOperand(3).setIsDead();
1370   }
1371 }
1372 
1373 // FIXME: Get this from tablegen.
get64BitArgumentGPRs(CallingConv::ID CallConv,const X86Subtarget & Subtarget)1374 static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
1375                                                 const X86Subtarget &Subtarget) {
1376   assert(Subtarget.is64Bit());
1377 
1378   if (Subtarget.isCallingConvWin64(CallConv)) {
1379     static const MCPhysReg GPR64ArgRegsWin64[] = {
1380       X86::RCX, X86::RDX, X86::R8,  X86::R9
1381     };
1382     return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
1383   }
1384 
1385   static const MCPhysReg GPR64ArgRegs64Bit[] = {
1386     X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1387   };
1388   return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
1389 }
1390 
has128ByteRedZone(const MachineFunction & MF) const1391 bool X86FrameLowering::has128ByteRedZone(const MachineFunction& MF) const {
1392   // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be
1393   // clobbered by any interrupt handler.
1394   assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1395          "MF used frame lowering for wrong subtarget");
1396   const Function &Fn = MF.getFunction();
1397   const bool IsWin64CC = STI.isCallingConvWin64(Fn.getCallingConv());
1398   return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Attribute::NoRedZone);
1399 }
1400 
1401 /// Return true if we need to use the restricted Windows x64 prologue and
1402 /// epilogue code patterns that can be described with WinCFI (.seh_*
1403 /// directives).
isWin64Prologue(const MachineFunction & MF) const1404 bool X86FrameLowering::isWin64Prologue(const MachineFunction &MF) const {
1405   return MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
1406 }
1407 
needsDwarfCFI(const MachineFunction & MF) const1408 bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const {
1409   return !isWin64Prologue(MF) && MF.needsFrameMoves();
1410 }
1411 
1412 /// emitPrologue - Push callee-saved registers onto the stack, which
1413 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate
1414 /// space for local variables. Also emit labels used by the exception handler to
1415 /// generate the exception handling frames.
1416 
1417 /*
1418   Here's a gist of what gets emitted:
1419 
1420   ; Establish frame pointer, if needed
1421   [if needs FP]
1422       push  %rbp
1423       .cfi_def_cfa_offset 16
1424       .cfi_offset %rbp, -16
1425       .seh_pushreg %rpb
1426       mov  %rsp, %rbp
1427       .cfi_def_cfa_register %rbp
1428 
1429   ; Spill general-purpose registers
1430   [for all callee-saved GPRs]
1431       pushq %<reg>
1432       [if not needs FP]
1433          .cfi_def_cfa_offset (offset from RETADDR)
1434       .seh_pushreg %<reg>
1435 
1436   ; If the required stack alignment > default stack alignment
1437   ; rsp needs to be re-aligned.  This creates a "re-alignment gap"
1438   ; of unknown size in the stack frame.
1439   [if stack needs re-alignment]
1440       and  $MASK, %rsp
1441 
1442   ; Allocate space for locals
1443   [if target is Windows and allocated space > 4096 bytes]
1444       ; Windows needs special care for allocations larger
1445       ; than one page.
1446       mov $NNN, %rax
1447       call ___chkstk_ms/___chkstk
1448       sub  %rax, %rsp
1449   [else]
1450       sub  $NNN, %rsp
1451 
1452   [if needs FP]
1453       .seh_stackalloc (size of XMM spill slots)
1454       .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
1455   [else]
1456       .seh_stackalloc NNN
1457 
1458   ; Spill XMMs
1459   ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
1460   ; they may get spilled on any platform, if the current function
1461   ; calls @llvm.eh.unwind.init
1462   [if needs FP]
1463       [for all callee-saved XMM registers]
1464           movaps  %<xmm reg>, -MMM(%rbp)
1465       [for all callee-saved XMM registers]
1466           .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
1467               ; i.e. the offset relative to (%rbp - SEHFrameOffset)
1468   [else]
1469       [for all callee-saved XMM registers]
1470           movaps  %<xmm reg>, KKK(%rsp)
1471       [for all callee-saved XMM registers]
1472           .seh_savexmm %<xmm reg>, KKK
1473 
1474   .seh_endprologue
1475 
1476   [if needs base pointer]
1477       mov  %rsp, %rbx
1478       [if needs to restore base pointer]
1479           mov %rsp, -MMM(%rbp)
1480 
1481   ; Emit CFI info
1482   [if needs FP]
1483       [for all callee-saved registers]
1484           .cfi_offset %<reg>, (offset from %rbp)
1485   [else]
1486        .cfi_def_cfa_offset (offset from RETADDR)
1487       [for all callee-saved registers]
1488           .cfi_offset %<reg>, (offset from %rsp)
1489 
1490   Notes:
1491   - .seh directives are emitted only for Windows 64 ABI
1492   - .cv_fpo directives are emitted on win32 when emitting CodeView
1493   - .cfi directives are emitted for all other ABIs
1494   - for 32-bit code, substitute %e?? registers for %r??
1495 */
1496 
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const1497 void X86FrameLowering::emitPrologue(MachineFunction &MF,
1498                                     MachineBasicBlock &MBB) const {
1499   assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1500          "MF used frame lowering for wrong subtarget");
1501   MachineBasicBlock::iterator MBBI = MBB.begin();
1502   MachineFrameInfo &MFI = MF.getFrameInfo();
1503   const Function &Fn = MF.getFunction();
1504   MachineModuleInfo &MMI = MF.getMMI();
1505   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1506   uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
1507   uint64_t StackSize = MFI.getStackSize();    // Number of bytes to allocate.
1508   bool IsFunclet = MBB.isEHFuncletEntry();
1509   EHPersonality Personality = EHPersonality::Unknown;
1510   if (Fn.hasPersonalityFn())
1511     Personality = classifyEHPersonality(Fn.getPersonalityFn());
1512   bool FnHasClrFunclet =
1513       MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR;
1514   bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1515   bool HasFP = hasFP(MF);
1516   bool IsWin64Prologue = isWin64Prologue(MF);
1517   bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
1518   // FIXME: Emit FPO data for EH funclets.
1519   bool NeedsWinFPO =
1520       !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag();
1521   bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1522   bool NeedsDwarfCFI = needsDwarfCFI(MF);
1523   Register FramePtr = TRI->getFrameRegister(MF);
1524   const Register MachineFramePtr =
1525       STI.isTarget64BitILP32()
1526           ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr;
1527   Register BasePtr = TRI->getBaseRegister();
1528   bool HasWinCFI = false;
1529 
1530   // Debug location must be unknown since the first debug location is used
1531   // to determine the end of the prologue.
1532   DebugLoc DL;
1533 
1534   // Space reserved for stack-based arguments when making a (ABI-guaranteed)
1535   // tail call.
1536   unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
1537   if (TailCallArgReserveSize  && IsWin64Prologue)
1538     report_fatal_error("Can't handle guaranteed tail call under win64 yet");
1539 
1540   const bool EmitStackProbeCall =
1541       STI.getTargetLowering()->hasStackProbeSymbol(MF);
1542   unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF);
1543 
1544   if (HasFP && X86FI->hasSwiftAsyncContext()) {
1545     switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1546     case SwiftAsyncFramePointerMode::DeploymentBased:
1547       if (STI.swiftAsyncContextIsDynamicallySet()) {
1548         // The special symbol below is absolute and has a *value* suitable to be
1549         // combined with the frame pointer directly.
1550         BuildMI(MBB, MBBI, DL, TII.get(X86::OR64rm), MachineFramePtr)
1551             .addUse(MachineFramePtr)
1552             .addUse(X86::RIP)
1553             .addImm(1)
1554             .addUse(X86::NoRegister)
1555             .addExternalSymbol("swift_async_extendedFramePointerFlags",
1556                                X86II::MO_GOTPCREL)
1557             .addUse(X86::NoRegister);
1558         break;
1559       }
1560       [[fallthrough]];
1561 
1562     case SwiftAsyncFramePointerMode::Always:
1563       BuildMI(MBB, MBBI, DL, TII.get(X86::BTS64ri8), MachineFramePtr)
1564           .addUse(MachineFramePtr)
1565           .addImm(60)
1566           .setMIFlag(MachineInstr::FrameSetup);
1567       break;
1568 
1569     case SwiftAsyncFramePointerMode::Never:
1570       break;
1571     }
1572   }
1573 
1574   // Re-align the stack on 64-bit if the x86-interrupt calling convention is
1575   // used and an error code was pushed, since the x86-64 ABI requires a 16-byte
1576   // stack alignment.
1577   if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit &&
1578       Fn.arg_size() == 2) {
1579     StackSize += 8;
1580     MFI.setStackSize(StackSize);
1581 
1582     // Update the stack pointer by pushing a register. This is the instruction
1583     // emitted that would be end up being emitted by a call to `emitSPUpdate`.
1584     // Hard-coding the update to a push avoids emitting a second
1585     // `STACKALLOC_W_PROBING` instruction in the save block: We know that stack
1586     // probing isn't needed anyways for an 8-byte update.
1587     // Pushing a register leaves us in a similar situation to a regular
1588     // function call where we know that the address at (rsp-8) is writeable.
1589     // That way we avoid any off-by-ones with stack probing for additional
1590     // stack pointer updates later on.
1591     BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1592         .addReg(X86::RAX, RegState::Undef)
1593         .setMIFlag(MachineInstr::FrameSetup);
1594   }
1595 
1596   // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
1597   // function, and use up to 128 bytes of stack space, don't have a frame
1598   // pointer, calls, or dynamic alloca then we do not need to adjust the
1599   // stack pointer (we fit in the Red Zone). We also check that we don't
1600   // push and pop from the stack.
1601   if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) &&
1602       !MFI.hasVarSizedObjects() &&             // No dynamic alloca.
1603       !MFI.adjustsStack() &&                   // No calls.
1604       !EmitStackProbeCall &&                   // No stack probes.
1605       !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop.
1606       !MF.shouldSplitStack()) {                // Regular stack
1607     uint64_t MinSize =
1608         X86FI->getCalleeSavedFrameSize() - X86FI->getTCReturnAddrDelta();
1609     if (HasFP) MinSize += SlotSize;
1610     X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0);
1611     StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
1612     MFI.setStackSize(StackSize);
1613   }
1614 
1615   // Insert stack pointer adjustment for later moving of return addr.  Only
1616   // applies to tail call optimized functions where the callee argument stack
1617   // size is bigger than the callers.
1618   if (TailCallArgReserveSize != 0) {
1619     BuildStackAdjustment(MBB, MBBI, DL, -(int)TailCallArgReserveSize,
1620                          /*InEpilogue=*/false)
1621         .setMIFlag(MachineInstr::FrameSetup);
1622   }
1623 
1624   // Mapping for machine moves:
1625   //
1626   //   DST: VirtualFP AND
1627   //        SRC: VirtualFP              => DW_CFA_def_cfa_offset
1628   //        ELSE                        => DW_CFA_def_cfa
1629   //
1630   //   SRC: VirtualFP AND
1631   //        DST: Register               => DW_CFA_def_cfa_register
1632   //
1633   //   ELSE
1634   //        OFFSET < 0                  => DW_CFA_offset_extended_sf
1635   //        REG < 64                    => DW_CFA_offset + Reg
1636   //        ELSE                        => DW_CFA_offset_extended
1637 
1638   uint64_t NumBytes = 0;
1639   int stackGrowth = -SlotSize;
1640 
1641   // Find the funclet establisher parameter
1642   Register Establisher = X86::NoRegister;
1643   if (IsClrFunclet)
1644     Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
1645   else if (IsFunclet)
1646     Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1647 
1648   if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1649     // Immediately spill establisher into the home slot.
1650     // The runtime cares about this.
1651     // MOV64mr %rdx, 16(%rsp)
1652     unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1653     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16)
1654         .addReg(Establisher)
1655         .setMIFlag(MachineInstr::FrameSetup);
1656     MBB.addLiveIn(Establisher);
1657   }
1658 
1659   if (HasFP) {
1660     assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved");
1661 
1662     // Calculate required stack adjustment.
1663     uint64_t FrameSize = StackSize - SlotSize;
1664     // If required, include space for extra hidden slot for stashing base pointer.
1665     if (X86FI->getRestoreBasePointer())
1666       FrameSize += SlotSize;
1667 
1668     NumBytes = FrameSize -
1669                (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1670 
1671     // Callee-saved registers are pushed on stack before the stack is realigned.
1672     if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1673       NumBytes = alignTo(NumBytes, MaxAlign);
1674 
1675     // Save EBP/RBP into the appropriate stack slot.
1676     BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
1677       .addReg(MachineFramePtr, RegState::Kill)
1678       .setMIFlag(MachineInstr::FrameSetup);
1679 
1680     if (NeedsDwarfCFI) {
1681       // Mark the place where EBP/RBP was saved.
1682       // Define the current CFA rule to use the provided offset.
1683       assert(StackSize);
1684       BuildCFI(MBB, MBBI, DL,
1685                MCCFIInstruction::cfiDefCfaOffset(nullptr, -2 * stackGrowth),
1686                MachineInstr::FrameSetup);
1687 
1688       // Change the rule for the FramePtr to be an "offset" rule.
1689       unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1690       BuildCFI(MBB, MBBI, DL,
1691                MCCFIInstruction::createOffset(nullptr, DwarfFramePtr,
1692                                               2 * stackGrowth),
1693                MachineInstr::FrameSetup);
1694     }
1695 
1696     if (NeedsWinCFI) {
1697       HasWinCFI = true;
1698       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1699           .addImm(FramePtr)
1700           .setMIFlag(MachineInstr::FrameSetup);
1701     }
1702 
1703     if (!IsFunclet) {
1704       if (X86FI->hasSwiftAsyncContext()) {
1705         const auto &Attrs = MF.getFunction().getAttributes();
1706 
1707         // Before we update the live frame pointer we have to ensure there's a
1708         // valid (or null) asynchronous context in its slot just before FP in
1709         // the frame record, so store it now.
1710         if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) {
1711           // We have an initial context in r14, store it just before the frame
1712           // pointer.
1713           MBB.addLiveIn(X86::R14);
1714           BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1715               .addReg(X86::R14)
1716               .setMIFlag(MachineInstr::FrameSetup);
1717         } else {
1718           // No initial context, store null so that there's no pointer that
1719           // could be misused.
1720           BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64i8))
1721               .addImm(0)
1722               .setMIFlag(MachineInstr::FrameSetup);
1723         }
1724 
1725         if (NeedsWinCFI) {
1726           HasWinCFI = true;
1727           BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1728               .addImm(X86::R14)
1729               .setMIFlag(MachineInstr::FrameSetup);
1730         }
1731 
1732         BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr)
1733             .addUse(X86::RSP)
1734             .addImm(1)
1735             .addUse(X86::NoRegister)
1736             .addImm(8)
1737             .addUse(X86::NoRegister)
1738             .setMIFlag(MachineInstr::FrameSetup);
1739         BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64ri8), X86::RSP)
1740             .addUse(X86::RSP)
1741             .addImm(8)
1742             .setMIFlag(MachineInstr::FrameSetup);
1743       }
1744 
1745       if (!IsWin64Prologue && !IsFunclet) {
1746         // Update EBP with the new base value.
1747         if (!X86FI->hasSwiftAsyncContext())
1748           BuildMI(MBB, MBBI, DL,
1749                   TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
1750                   FramePtr)
1751               .addReg(StackPtr)
1752               .setMIFlag(MachineInstr::FrameSetup);
1753 
1754       if (SaveArgs && !Fn.arg_empty()) {
1755         ArrayRef<MCPhysReg> GPRs =
1756           get64BitArgumentGPRs(Fn.getCallingConv(), STI);
1757         unsigned arg_size = Fn.arg_size();
1758         unsigned RI = 0;
1759         int64_t SaveSize = 0;
1760 
1761         if (Fn.hasStructRetAttr()) {
1762           GPRs = GPRs.drop_front(1);
1763           arg_size--;
1764         }
1765 
1766         for (MCPhysReg Reg : GPRs) {
1767           if (++RI > arg_size)
1768             break;
1769 
1770           SaveSize += SlotSize;
1771 
1772           BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1773             .addReg(Reg)
1774             .setMIFlag(MachineInstr::FrameSetup);
1775         }
1776 
1777         // Realign the stack. PUSHes are the most space efficient.
1778         while (SaveSize % getStackAlignment()) {
1779           BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1780             .addReg(GPRs.front())
1781             .setMIFlag(MachineInstr::FrameSetup);
1782 
1783           SaveSize += SlotSize;
1784         }
1785 
1786         //dlg StackSize -= SaveSize;
1787         //dlg MFI.setStackSize(StackSize);
1788         X86FI->setSaveArgSize(SaveSize);
1789       }
1790 
1791         if (NeedsDwarfCFI) {
1792           // Mark effective beginning of when frame pointer becomes valid.
1793           // Define the current CFA to use the EBP/RBP register.
1794           unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1795           BuildCFI(
1796               MBB, MBBI, DL,
1797               MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr),
1798               MachineInstr::FrameSetup);
1799         }
1800 
1801         if (NeedsWinFPO) {
1802           // .cv_fpo_setframe $FramePtr
1803           HasWinCFI = true;
1804           BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
1805               .addImm(FramePtr)
1806               .addImm(0)
1807               .setMIFlag(MachineInstr::FrameSetup);
1808         }
1809       }
1810     }
1811   } else {
1812     assert(!IsFunclet && "funclets without FPs not yet implemented");
1813     NumBytes = StackSize -
1814                (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1815   }
1816 
1817   // Update the offset adjustment, which is mainly used by codeview to translate
1818   // from ESP to VFRAME relative local variable offsets.
1819   if (!IsFunclet) {
1820     if (HasFP && TRI->hasStackRealignment(MF))
1821       MFI.setOffsetAdjustment(-NumBytes);
1822     else
1823       MFI.setOffsetAdjustment(-StackSize);
1824   }
1825 
1826   // For EH funclets, only allocate enough space for outgoing calls. Save the
1827   // NumBytes value that we would've used for the parent frame.
1828   unsigned ParentFrameNumBytes = NumBytes;
1829   if (IsFunclet)
1830     NumBytes = getWinEHFuncletFrameSize(MF);
1831 
1832   // Skip the callee-saved push instructions.
1833   bool PushedRegs = false;
1834   int StackOffset = 2 * stackGrowth;
1835 
1836   while (MBBI != MBB.end() &&
1837          MBBI->getFlag(MachineInstr::FrameSetup) &&
1838          (MBBI->getOpcode() == X86::PUSH32r ||
1839           MBBI->getOpcode() == X86::PUSH64r)) {
1840     PushedRegs = true;
1841     Register Reg = MBBI->getOperand(0).getReg();
1842     ++MBBI;
1843 
1844     if (!HasFP && NeedsDwarfCFI) {
1845       // Mark callee-saved push instruction.
1846       // Define the current CFA rule to use the provided offset.
1847       assert(StackSize);
1848       BuildCFI(MBB, MBBI, DL,
1849                MCCFIInstruction::cfiDefCfaOffset(nullptr, -StackOffset),
1850                MachineInstr::FrameSetup);
1851       StackOffset += stackGrowth;
1852     }
1853 
1854     if (NeedsWinCFI) {
1855       HasWinCFI = true;
1856       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1857           .addImm(Reg)
1858           .setMIFlag(MachineInstr::FrameSetup);
1859     }
1860   }
1861 
1862   // Realign stack after we pushed callee-saved registers (so that we'll be
1863   // able to calculate their offsets from the frame pointer).
1864   // Don't do this for Win64, it needs to realign the stack after the prologue.
1865   if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF)) {
1866     assert(HasFP && "There should be a frame pointer if stack is realigned.");
1867     BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign);
1868 
1869     if (NeedsWinCFI) {
1870       HasWinCFI = true;
1871       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlign))
1872           .addImm(MaxAlign)
1873           .setMIFlag(MachineInstr::FrameSetup);
1874     }
1875   }
1876 
1877   // If there is an SUB32ri of ESP immediately before this instruction, merge
1878   // the two. This can be the case when tail call elimination is enabled and
1879   // the callee has more arguments then the caller.
1880   NumBytes -= mergeSPUpdates(MBB, MBBI, true);
1881 
1882   // Adjust stack pointer: ESP -= numbytes.
1883 
1884   // Windows and cygwin/mingw require a prologue helper routine when allocating
1885   // more than 4K bytes on the stack.  Windows uses __chkstk and cygwin/mingw
1886   // uses __alloca.  __alloca and the 32-bit version of __chkstk will probe the
1887   // stack and adjust the stack pointer in one go.  The 64-bit version of
1888   // __chkstk is only responsible for probing the stack.  The 64-bit prologue is
1889   // responsible for adjusting the stack pointer.  Touching the stack at 4K
1890   // increments is necessary to ensure that the guard pages used by the OS
1891   // virtual memory manager are allocated in correct sequence.
1892   uint64_t AlignedNumBytes = NumBytes;
1893   if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF))
1894     AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign);
1895   if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
1896     assert(!X86FI->getUsesRedZone() &&
1897            "The Red Zone is not accounted for in stack probes");
1898 
1899     // Check whether EAX is livein for this block.
1900     bool isEAXAlive = isEAXLiveIn(MBB);
1901 
1902     if (isEAXAlive) {
1903       if (Is64Bit) {
1904         // Save RAX
1905         BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1906           .addReg(X86::RAX, RegState::Kill)
1907           .setMIFlag(MachineInstr::FrameSetup);
1908       } else {
1909         // Save EAX
1910         BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
1911           .addReg(X86::EAX, RegState::Kill)
1912           .setMIFlag(MachineInstr::FrameSetup);
1913       }
1914     }
1915 
1916     if (Is64Bit) {
1917       // Handle the 64-bit Windows ABI case where we need to call __chkstk.
1918       // Function prologue is responsible for adjusting the stack pointer.
1919       int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
1920       BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Alloc)), X86::RAX)
1921           .addImm(Alloc)
1922           .setMIFlag(MachineInstr::FrameSetup);
1923     } else {
1924       // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
1925       // We'll also use 4 already allocated bytes for EAX.
1926       BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
1927           .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1928           .setMIFlag(MachineInstr::FrameSetup);
1929     }
1930 
1931     // Call __chkstk, __chkstk_ms, or __alloca.
1932     emitStackProbe(MF, MBB, MBBI, DL, true);
1933 
1934     if (isEAXAlive) {
1935       // Restore RAX/EAX
1936       MachineInstr *MI;
1937       if (Is64Bit)
1938         MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV64rm), X86::RAX),
1939                           StackPtr, false, NumBytes - 8);
1940       else
1941         MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX),
1942                           StackPtr, false, NumBytes - 4);
1943       MI->setFlag(MachineInstr::FrameSetup);
1944       MBB.insert(MBBI, MI);
1945     }
1946   } else if (NumBytes) {
1947     emitSPUpdate(MBB, MBBI, DL, -(int64_t)NumBytes, /*InEpilogue=*/false);
1948   }
1949 
1950   if (NeedsWinCFI && NumBytes) {
1951     HasWinCFI = true;
1952     BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
1953         .addImm(NumBytes)
1954         .setMIFlag(MachineInstr::FrameSetup);
1955   }
1956 
1957   int SEHFrameOffset = 0;
1958   unsigned SPOrEstablisher;
1959   if (IsFunclet) {
1960     if (IsClrFunclet) {
1961       // The establisher parameter passed to a CLR funclet is actually a pointer
1962       // to the (mostly empty) frame of its nearest enclosing funclet; we have
1963       // to find the root function establisher frame by loading the PSPSym from
1964       // the intermediate frame.
1965       unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
1966       MachinePointerInfo NoInfo;
1967       MBB.addLiveIn(Establisher);
1968       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher),
1969                    Establisher, false, PSPSlotOffset)
1970           .addMemOperand(MF.getMachineMemOperand(
1971               NoInfo, MachineMemOperand::MOLoad, SlotSize, Align(SlotSize)));
1972       ;
1973       // Save the root establisher back into the current funclet's (mostly
1974       // empty) frame, in case a sub-funclet or the GC needs it.
1975       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr,
1976                    false, PSPSlotOffset)
1977           .addReg(Establisher)
1978           .addMemOperand(MF.getMachineMemOperand(
1979               NoInfo,
1980               MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
1981               SlotSize, Align(SlotSize)));
1982     }
1983     SPOrEstablisher = Establisher;
1984   } else {
1985     SPOrEstablisher = StackPtr;
1986   }
1987 
1988   if (IsWin64Prologue && HasFP) {
1989     // Set RBP to a small fixed offset from RSP. In the funclet case, we base
1990     // this calculation on the incoming establisher, which holds the value of
1991     // RSP from the parent frame at the end of the prologue.
1992     SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes);
1993     if (SEHFrameOffset)
1994       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
1995                    SPOrEstablisher, false, SEHFrameOffset);
1996     else
1997       BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr)
1998           .addReg(SPOrEstablisher);
1999 
2000     // If this is not a funclet, emit the CFI describing our frame pointer.
2001     if (NeedsWinCFI && !IsFunclet) {
2002       assert(!NeedsWinFPO && "this setframe incompatible with FPO data");
2003       HasWinCFI = true;
2004       BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
2005           .addImm(FramePtr)
2006           .addImm(SEHFrameOffset)
2007           .setMIFlag(MachineInstr::FrameSetup);
2008       if (isAsynchronousEHPersonality(Personality))
2009         MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset;
2010     }
2011   } else if (IsFunclet && STI.is32Bit()) {
2012     // Reset EBP / ESI to something good for funclets.
2013     MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL);
2014     // If we're a catch funclet, we can be returned to via catchret. Save ESP
2015     // into the registration node so that the runtime will restore it for us.
2016     if (!MBB.isCleanupFuncletEntry()) {
2017       assert(Personality == EHPersonality::MSVC_CXX);
2018       Register FrameReg;
2019       int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex;
2020       int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed();
2021       // ESP is the first field, so no extra displacement is needed.
2022       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg,
2023                    false, EHRegOffset)
2024           .addReg(X86::ESP);
2025     }
2026   }
2027 
2028   while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) {
2029     const MachineInstr &FrameInstr = *MBBI;
2030     ++MBBI;
2031 
2032     if (NeedsWinCFI) {
2033       int FI;
2034       if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) {
2035         if (X86::FR64RegClass.contains(Reg)) {
2036           int Offset;
2037           Register IgnoredFrameReg;
2038           if (IsWin64Prologue && IsFunclet)
2039             Offset = getWin64EHFrameIndexRef(MF, FI, IgnoredFrameReg);
2040           else
2041             Offset =
2042                 getFrameIndexReference(MF, FI, IgnoredFrameReg).getFixed() +
2043                 SEHFrameOffset;
2044 
2045           HasWinCFI = true;
2046           assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data");
2047           BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM))
2048               .addImm(Reg)
2049               .addImm(Offset)
2050               .setMIFlag(MachineInstr::FrameSetup);
2051         }
2052       }
2053     }
2054   }
2055 
2056   if (NeedsWinCFI && HasWinCFI)
2057     BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
2058         .setMIFlag(MachineInstr::FrameSetup);
2059 
2060   if (FnHasClrFunclet && !IsFunclet) {
2061     // Save the so-called Initial-SP (i.e. the value of the stack pointer
2062     // immediately after the prolog)  into the PSPSlot so that funclets
2063     // and the GC can recover it.
2064     unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2065     auto PSPInfo = MachinePointerInfo::getFixedStack(
2066         MF, MF.getWinEHFuncInfo()->PSPSymFrameIdx);
2067     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false,
2068                  PSPSlotOffset)
2069         .addReg(StackPtr)
2070         .addMemOperand(MF.getMachineMemOperand(
2071             PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2072             SlotSize, Align(SlotSize)));
2073   }
2074 
2075   // Realign stack after we spilled callee-saved registers (so that we'll be
2076   // able to calculate their offsets from the frame pointer).
2077   // Win64 requires aligning the stack after the prologue.
2078   if (IsWin64Prologue && TRI->hasStackRealignment(MF)) {
2079     assert(HasFP && "There should be a frame pointer if stack is realigned.");
2080     BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign);
2081   }
2082 
2083   // We already dealt with stack realignment and funclets above.
2084   if (IsFunclet && STI.is32Bit())
2085     return;
2086 
2087   // If we need a base pointer, set it up here. It's whatever the value
2088   // of the stack pointer is at this point. Any variable size objects
2089   // will be allocated after this, so we can still use the base pointer
2090   // to reference locals.
2091   if (TRI->hasBasePointer(MF)) {
2092     // Update the base pointer with the current stack pointer.
2093     unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
2094     BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
2095       .addReg(SPOrEstablisher)
2096       .setMIFlag(MachineInstr::FrameSetup);
2097     if (X86FI->getRestoreBasePointer()) {
2098       // Stash value of base pointer.  Saving RSP instead of EBP shortens
2099       // dependence chain. Used by SjLj EH.
2100       unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2101       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)),
2102                    FramePtr, true, X86FI->getRestoreBasePointerOffset())
2103         .addReg(SPOrEstablisher)
2104         .setMIFlag(MachineInstr::FrameSetup);
2105     }
2106 
2107     if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
2108       // Stash the value of the frame pointer relative to the base pointer for
2109       // Win32 EH. This supports Win32 EH, which does the inverse of the above:
2110       // it recovers the frame pointer from the base pointer rather than the
2111       // other way around.
2112       unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2113       Register UsedReg;
2114       int Offset =
2115           getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg)
2116               .getFixed();
2117       assert(UsedReg == BasePtr);
2118       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset)
2119           .addReg(FramePtr)
2120           .setMIFlag(MachineInstr::FrameSetup);
2121     }
2122   }
2123 
2124   if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2125     // Mark end of stack pointer adjustment.
2126     if (!HasFP && NumBytes) {
2127       // Define the current CFA rule to use the provided offset.
2128       assert(StackSize);
2129       BuildCFI(
2130           MBB, MBBI, DL,
2131           MCCFIInstruction::cfiDefCfaOffset(nullptr, StackSize - stackGrowth),
2132           MachineInstr::FrameSetup);
2133     }
2134 
2135     // Emit DWARF info specifying the offsets of the callee-saved registers.
2136     emitCalleeSavedFrameMoves(MBB, MBBI, DL, true);
2137   }
2138 
2139   // X86 Interrupt handling function cannot assume anything about the direction
2140   // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction
2141   // in each prologue of interrupt handler function.
2142   //
2143   // FIXME: Create "cld" instruction only in these cases:
2144   // 1. The interrupt handling function uses any of the "rep" instructions.
2145   // 2. Interrupt handling function calls another function.
2146   //
2147   if (Fn.getCallingConv() == CallingConv::X86_INTR)
2148     BuildMI(MBB, MBBI, DL, TII.get(X86::CLD))
2149         .setMIFlag(MachineInstr::FrameSetup);
2150 
2151   // At this point we know if the function has WinCFI or not.
2152   MF.setHasWinCFI(HasWinCFI);
2153 }
2154 
canUseLEAForSPInEpilogue(const MachineFunction & MF) const2155 bool X86FrameLowering::canUseLEAForSPInEpilogue(
2156     const MachineFunction &MF) const {
2157   // We can't use LEA instructions for adjusting the stack pointer if we don't
2158   // have a frame pointer in the Win64 ABI.  Only ADD instructions may be used
2159   // to deallocate the stack.
2160   // This means that we can use LEA for SP in two situations:
2161   // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
2162   // 2. We *have* a frame pointer which means we are permitted to use LEA.
2163   return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF);
2164 }
2165 
isFuncletReturnInstr(MachineInstr & MI)2166 static bool isFuncletReturnInstr(MachineInstr &MI) {
2167   switch (MI.getOpcode()) {
2168   case X86::CATCHRET:
2169   case X86::CLEANUPRET:
2170     return true;
2171   default:
2172     return false;
2173   }
2174   llvm_unreachable("impossible");
2175 }
2176 
2177 // CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
2178 // stack. It holds a pointer to the bottom of the root function frame.  The
2179 // establisher frame pointer passed to a nested funclet may point to the
2180 // (mostly empty) frame of its parent funclet, but it will need to find
2181 // the frame of the root function to access locals.  To facilitate this,
2182 // every funclet copies the pointer to the bottom of the root function
2183 // frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
2184 // same offset for the PSPSym in the root function frame that's used in the
2185 // funclets' frames allows each funclet to dynamically accept any ancestor
2186 // frame as its establisher argument (the runtime doesn't guarantee the
2187 // immediate parent for some reason lost to history), and also allows the GC,
2188 // which uses the PSPSym for some bookkeeping, to find it in any funclet's
2189 // frame with only a single offset reported for the entire method.
2190 unsigned
getPSPSlotOffsetFromSP(const MachineFunction & MF) const2191 X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
2192   const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
2193   Register SPReg;
2194   int Offset = getFrameIndexReferencePreferSP(MF, Info.PSPSymFrameIdx, SPReg,
2195                                               /*IgnoreSPUpdates*/ true)
2196                    .getFixed();
2197   assert(Offset >= 0 && SPReg == TRI->getStackRegister());
2198   return static_cast<unsigned>(Offset);
2199 }
2200 
2201 unsigned
getWinEHFuncletFrameSize(const MachineFunction & MF) const2202 X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
2203   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2204   // This is the size of the pushed CSRs.
2205   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2206   // This is the size of callee saved XMMs.
2207   const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2208   unsigned XMMSize = WinEHXMMSlotInfo.size() *
2209                      TRI->getSpillSize(X86::VR128RegClass);
2210   // This is the amount of stack a funclet needs to allocate.
2211   unsigned UsedSize;
2212   EHPersonality Personality =
2213       classifyEHPersonality(MF.getFunction().getPersonalityFn());
2214   if (Personality == EHPersonality::CoreCLR) {
2215     // CLR funclets need to hold enough space to include the PSPSym, at the
2216     // same offset from the stack pointer (immediately after the prolog) as it
2217     // resides at in the main function.
2218     UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
2219   } else {
2220     // Other funclets just need enough stack for outgoing call arguments.
2221     UsedSize = MF.getFrameInfo().getMaxCallFrameSize();
2222   }
2223   // RBP is not included in the callee saved register block. After pushing RBP,
2224   // everything is 16 byte aligned. Everything we allocate before an outgoing
2225   // call must also be 16 byte aligned.
2226   unsigned FrameSizeMinusRBP = alignTo(CSSize + UsedSize, getStackAlign());
2227   // Subtract out the size of the callee saved registers. This is how much stack
2228   // each funclet will allocate.
2229   return FrameSizeMinusRBP + XMMSize - CSSize;
2230 }
2231 
isTailCallOpcode(unsigned Opc)2232 static bool isTailCallOpcode(unsigned Opc) {
2233     return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
2234         Opc == X86::TCRETURNmi ||
2235         Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 ||
2236         Opc == X86::TCRETURNmi64;
2237 }
2238 
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const2239 void X86FrameLowering::emitEpilogue(MachineFunction &MF,
2240                                     MachineBasicBlock &MBB) const {
2241   const MachineFrameInfo &MFI = MF.getFrameInfo();
2242   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2243   MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator();
2244   MachineBasicBlock::iterator MBBI = Terminator;
2245   DebugLoc DL;
2246   if (MBBI != MBB.end())
2247     DL = MBBI->getDebugLoc();
2248   // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
2249   const bool Is64BitILP32 = STI.isTarget64BitILP32();
2250   Register FramePtr = TRI->getFrameRegister(MF);
2251   Register MachineFramePtr =
2252       Is64BitILP32 ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr;
2253 
2254   bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2255   bool NeedsWin64CFI =
2256       IsWin64Prologue && MF.getFunction().needsUnwindTableEntry();
2257   bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(*MBBI);
2258 
2259   // Get the number of bytes to allocate from the FrameInfo.
2260   uint64_t StackSize = MFI.getStackSize();
2261   uint64_t MaxAlign = calculateMaxStackAlign(MF);
2262   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2263   unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
2264   bool HasFP = hasFP(MF);
2265   uint64_t NumBytes = 0;
2266 
2267   bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() &&
2268                         !MF.getTarget().getTargetTriple().isOSWindows()) &&
2269                        MF.needsFrameMoves();
2270 
2271   if (IsFunclet) {
2272     assert(HasFP && "EH funclets without FP not yet implemented");
2273     NumBytes = getWinEHFuncletFrameSize(MF);
2274   } else if (HasFP) {
2275     // Calculate required stack adjustment.
2276     uint64_t FrameSize = StackSize - SlotSize;
2277     NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2278 
2279     // Callee-saved registers were pushed on stack before the stack was
2280     // realigned.
2281     if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2282       NumBytes = alignTo(FrameSize, MaxAlign);
2283   } else {
2284     NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2285   }
2286   uint64_t SEHStackAllocAmt = NumBytes;
2287 
2288   // AfterPop is the position to insert .cfi_restore.
2289   MachineBasicBlock::iterator AfterPop = MBBI;
2290   if (HasFP) {
2291     if (X86FI->hasSwiftAsyncContext()) {
2292       // Discard the context.
2293       int Offset = 16 + mergeSPUpdates(MBB, MBBI, true);
2294       emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue*/true);
2295     }
2296 
2297     if (X86FI->getSaveArgSize()) {
2298       // LEAVE is effectively mov rbp,rsp; pop rbp
2299       BuildMI(MBB, MBBI, DL, TII.get(X86::LEAVE64))
2300         .setMIFlag(MachineInstr::FrameDestroy);
2301     } else {
2302       // Pop EBP.
2303       BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r),
2304             MachineFramePtr)
2305         .setMIFlag(MachineInstr::FrameDestroy);
2306     }
2307 
2308     // We need to reset FP to its untagged state on return. Bit 60 is currently
2309     // used to show the presence of an extended frame.
2310     if (X86FI->hasSwiftAsyncContext()) {
2311       BuildMI(MBB, MBBI, DL, TII.get(X86::BTR64ri8),
2312               MachineFramePtr)
2313           .addUse(MachineFramePtr)
2314           .addImm(60)
2315           .setMIFlag(MachineInstr::FrameDestroy);
2316     }
2317 
2318     if (NeedsDwarfCFI) {
2319       unsigned DwarfStackPtr =
2320           TRI->getDwarfRegNum(Is64Bit ? X86::RSP : X86::ESP, true);
2321       BuildCFI(MBB, MBBI, DL,
2322                MCCFIInstruction::cfiDefCfa(nullptr, DwarfStackPtr, SlotSize),
2323                MachineInstr::FrameDestroy);
2324       if (!MBB.succ_empty() && !MBB.isReturnBlock()) {
2325         unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
2326         BuildCFI(MBB, AfterPop, DL,
2327                  MCCFIInstruction::createRestore(nullptr, DwarfFramePtr),
2328                  MachineInstr::FrameDestroy);
2329         --MBBI;
2330         --AfterPop;
2331       }
2332       --MBBI;
2333     }
2334   }
2335 
2336   MachineBasicBlock::iterator FirstCSPop = MBBI;
2337   // Skip the callee-saved pop instructions.
2338   while (MBBI != MBB.begin()) {
2339     MachineBasicBlock::iterator PI = std::prev(MBBI);
2340     unsigned Opc = PI->getOpcode();
2341 
2342     if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2343       if ((Opc != X86::POP32r || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2344           (Opc != X86::POP64r || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2345           (Opc != X86::LEAVE64 || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2346           (Opc != X86::BTR64ri8 || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2347           (Opc != X86::ADD64ri8 || !PI->getFlag(MachineInstr::FrameDestroy)))
2348         break;
2349       FirstCSPop = PI;
2350     }
2351 
2352     --MBBI;
2353   }
2354   MBBI = FirstCSPop;
2355 
2356   if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2357     emitCatchRetReturnValue(MBB, FirstCSPop, &*Terminator);
2358 
2359   if (MBBI != MBB.end())
2360     DL = MBBI->getDebugLoc();
2361   // If there is an ADD32ri or SUB32ri of ESP immediately before this
2362   // instruction, merge the two instructions.
2363   if (NumBytes || MFI.hasVarSizedObjects())
2364     NumBytes += mergeSPUpdates(MBB, MBBI, true);
2365 
2366   // If dynamic alloca is used, then reset esp to point to the last callee-saved
2367   // slot before popping them off! Same applies for the case, when stack was
2368   // realigned. Don't do this if this was a funclet epilogue, since the funclets
2369   // will not do realignment or dynamic stack allocation.
2370   if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) &&
2371       !IsFunclet) {
2372     if (TRI->hasStackRealignment(MF))
2373       MBBI = FirstCSPop;
2374     unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt);
2375     uint64_t LEAAmount =
2376         IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2377 
2378     if (X86FI->hasSwiftAsyncContext())
2379       LEAAmount -= 16;
2380 
2381     // There are only two legal forms of epilogue:
2382     // - add SEHAllocationSize, %rsp
2383     // - lea SEHAllocationSize(%FramePtr), %rsp
2384     //
2385     // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
2386     // However, we may use this sequence if we have a frame pointer because the
2387     // effects of the prologue can safely be undone.
2388     if (LEAAmount != 0) {
2389       unsigned Opc = getLEArOpcode(Uses64BitFramePtr);
2390       addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr),
2391                    FramePtr, false, LEAAmount);
2392       --MBBI;
2393     } else {
2394       unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
2395       BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
2396         .addReg(FramePtr);
2397       --MBBI;
2398     }
2399   } else if (NumBytes) {
2400     // Adjust stack pointer back: ESP += numbytes.
2401     emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true);
2402     if (!HasFP && NeedsDwarfCFI) {
2403       // Define the current CFA rule to use the provided offset.
2404       BuildCFI(MBB, MBBI, DL,
2405                MCCFIInstruction::cfiDefCfaOffset(
2406                    nullptr, CSSize + TailCallArgReserveSize + SlotSize),
2407                MachineInstr::FrameDestroy);
2408     }
2409     --MBBI;
2410   }
2411 
2412   // Windows unwinder will not invoke function's exception handler if IP is
2413   // either in prologue or in epilogue.  This behavior causes a problem when a
2414   // call immediately precedes an epilogue, because the return address points
2415   // into the epilogue.  To cope with that, we insert an epilogue marker here,
2416   // then replace it with a 'nop' if it ends up immediately after a CALL in the
2417   // final emitted code.
2418   if (NeedsWin64CFI && MF.hasWinCFI())
2419     BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue));
2420 
2421   if (!HasFP && NeedsDwarfCFI) {
2422     MBBI = FirstCSPop;
2423     int64_t Offset = -CSSize - SlotSize;
2424     // Mark callee-saved pop instruction.
2425     // Define the current CFA rule to use the provided offset.
2426     while (MBBI != MBB.end()) {
2427       MachineBasicBlock::iterator PI = MBBI;
2428       unsigned Opc = PI->getOpcode();
2429       ++MBBI;
2430       if (Opc == X86::POP32r || Opc == X86::POP64r) {
2431         Offset += SlotSize;
2432         BuildCFI(MBB, MBBI, DL,
2433                  MCCFIInstruction::cfiDefCfaOffset(nullptr, -Offset),
2434                  MachineInstr::FrameDestroy);
2435       }
2436     }
2437   }
2438 
2439   // Emit DWARF info specifying the restores of the callee-saved registers.
2440   // For epilogue with return inside or being other block without successor,
2441   // no need to generate .cfi_restore for callee-saved registers.
2442   if (NeedsDwarfCFI && !MBB.succ_empty())
2443     emitCalleeSavedFrameMoves(MBB, AfterPop, DL, false);
2444 
2445   if (Terminator == MBB.end() || !isTailCallOpcode(Terminator->getOpcode())) {
2446     // Add the return addr area delta back since we are not tail calling.
2447     int Offset = -1 * X86FI->getTCReturnAddrDelta();
2448     assert(Offset >= 0 && "TCDelta should never be positive");
2449     if (Offset) {
2450       // Check for possible merge with preceding ADD instruction.
2451       Offset += mergeSPUpdates(MBB, Terminator, true);
2452       emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true);
2453     }
2454   }
2455 
2456   // Emit tilerelease for AMX kernel.
2457   if (X86FI->hasVirtualTileReg())
2458     BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
2459 }
2460 
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const2461 StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
2462                                                      int FI,
2463                                                      Register &FrameReg) const {
2464   const MachineFrameInfo &MFI = MF.getFrameInfo();
2465 
2466   bool IsFixed = MFI.isFixedObjectIndex(FI);
2467   // We can't calculate offset from frame pointer if the stack is realigned,
2468   // so enforce usage of stack/base pointer.  The base pointer is used when we
2469   // have dynamic allocas in addition to dynamic realignment.
2470   if (TRI->hasBasePointer(MF))
2471     FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister();
2472   else if (TRI->hasStackRealignment(MF))
2473     FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister();
2474   else
2475     FrameReg = TRI->getFrameRegister(MF);
2476 
2477   // Offset will hold the offset from the stack pointer at function entry to the
2478   // object.
2479   // We need to factor in additional offsets applied during the prologue to the
2480   // frame, base, and stack pointer depending on which is used.
2481   int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
2482   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2483   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2484   uint64_t StackSize = MFI.getStackSize();
2485   bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2486   int64_t FPDelta = 0;
2487 
2488   // In an x86 interrupt, remove the offset we added to account for the return
2489   // address from any stack object allocated in the caller's frame. Interrupts
2490   // do not have a standard return address. Fixed objects in the current frame,
2491   // such as SSE register spills, should not get this treatment.
2492   if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR &&
2493       Offset >= 0) {
2494     Offset += getOffsetOfLocalArea();
2495   }
2496 
2497   if (IsWin64Prologue) {
2498     assert(!MFI.hasCalls() || (StackSize % 16) == 8);
2499 
2500     // Calculate required stack adjustment.
2501     uint64_t FrameSize = StackSize - SlotSize;
2502     // If required, include space for extra hidden slot for stashing base pointer.
2503     if (X86FI->getRestoreBasePointer())
2504       FrameSize += SlotSize;
2505     uint64_t NumBytes = FrameSize - CSSize;
2506 
2507     uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes);
2508     if (FI && FI == X86FI->getFAIndex())
2509       return StackOffset::getFixed(-SEHFrameOffset);
2510 
2511     // FPDelta is the offset from the "traditional" FP location of the old base
2512     // pointer followed by return address and the location required by the
2513     // restricted Win64 prologue.
2514     // Add FPDelta to all offsets below that go through the frame pointer.
2515     FPDelta = FrameSize - SEHFrameOffset;
2516     assert((!MFI.hasCalls() || (FPDelta % 16) == 0) &&
2517            "FPDelta isn't aligned per the Win64 ABI!");
2518   }
2519 
2520   if (FI >= 0)
2521     Offset -= X86FI->getSaveArgSize();
2522 
2523   if (FrameReg == TRI->getFramePtr()) {
2524     // Skip saved EBP/RBP
2525     Offset += SlotSize;
2526 
2527     // Account for restricted Windows prologue.
2528     Offset += FPDelta;
2529 
2530     // Skip the RETADDR move area
2531     int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2532     if (TailCallReturnAddrDelta < 0)
2533       Offset -= TailCallReturnAddrDelta;
2534 
2535     return StackOffset::getFixed(Offset);
2536   }
2537 
2538   // FrameReg is either the stack pointer or a base pointer. But the base is
2539   // located at the end of the statically known StackSize so the distinction
2540   // doesn't really matter.
2541   if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF))
2542     assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
2543   return StackOffset::getFixed(Offset + StackSize);
2544 }
2545 
getWin64EHFrameIndexRef(const MachineFunction & MF,int FI,Register & FrameReg) const2546 int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, int FI,
2547                                               Register &FrameReg) const {
2548   const MachineFrameInfo &MFI = MF.getFrameInfo();
2549   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2550   const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2551   const auto it = WinEHXMMSlotInfo.find(FI);
2552 
2553   if (it == WinEHXMMSlotInfo.end())
2554     return getFrameIndexReference(MF, FI, FrameReg).getFixed();
2555 
2556   FrameReg = TRI->getStackRegister();
2557   return alignDown(MFI.getMaxCallFrameSize(), getStackAlign().value()) +
2558          it->second;
2559 }
2560 
2561 StackOffset
getFrameIndexReferenceSP(const MachineFunction & MF,int FI,Register & FrameReg,int Adjustment) const2562 X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, int FI,
2563                                            Register &FrameReg,
2564                                            int Adjustment) const {
2565   const MachineFrameInfo &MFI = MF.getFrameInfo();
2566   FrameReg = TRI->getStackRegister();
2567   return StackOffset::getFixed(MFI.getObjectOffset(FI) -
2568                                getOffsetOfLocalArea() + Adjustment);
2569 }
2570 
2571 StackOffset
getFrameIndexReferencePreferSP(const MachineFunction & MF,int FI,Register & FrameReg,bool IgnoreSPUpdates) const2572 X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF,
2573                                                  int FI, Register &FrameReg,
2574                                                  bool IgnoreSPUpdates) const {
2575 
2576   const MachineFrameInfo &MFI = MF.getFrameInfo();
2577   // Does not include any dynamic realign.
2578   const uint64_t StackSize = MFI.getStackSize();
2579   // LLVM arranges the stack as follows:
2580   //   ...
2581   //   ARG2
2582   //   ARG1
2583   //   RETADDR
2584   //   PUSH RBP   <-- RBP points here
2585   //   PUSH CSRs
2586   //   ~~~~~~~    <-- possible stack realignment (non-win64)
2587   //   ...
2588   //   STACK OBJECTS
2589   //   ...        <-- RSP after prologue points here
2590   //   ~~~~~~~    <-- possible stack realignment (win64)
2591   //
2592   // if (hasVarSizedObjects()):
2593   //   ...        <-- "base pointer" (ESI/RBX) points here
2594   //   DYNAMIC ALLOCAS
2595   //   ...        <-- RSP points here
2596   //
2597   // Case 1: In the simple case of no stack realignment and no dynamic
2598   // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
2599   // with fixed offsets from RSP.
2600   //
2601   // Case 2: In the case of stack realignment with no dynamic allocas, fixed
2602   // stack objects are addressed with RBP and regular stack objects with RSP.
2603   //
2604   // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
2605   // to address stack arguments for outgoing calls and nothing else. The "base
2606   // pointer" points to local variables, and RBP points to fixed objects.
2607   //
2608   // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
2609   // answer we give is relative to the SP after the prologue, and not the
2610   // SP in the middle of the function.
2611 
2612   if (MFI.isFixedObjectIndex(FI) && TRI->hasStackRealignment(MF) &&
2613       !STI.isTargetWin64())
2614     return getFrameIndexReference(MF, FI, FrameReg);
2615 
2616   // If !hasReservedCallFrame the function might have SP adjustement in the
2617   // body.  So, even though the offset is statically known, it depends on where
2618   // we are in the function.
2619   if (!IgnoreSPUpdates && !hasReservedCallFrame(MF))
2620     return getFrameIndexReference(MF, FI, FrameReg);
2621 
2622   // We don't handle tail calls, and shouldn't be seeing them either.
2623   assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 &&
2624          "we don't handle this case!");
2625 
2626   // This is how the math works out:
2627   //
2628   //  %rsp grows (i.e. gets lower) left to right. Each box below is
2629   //  one word (eight bytes).  Obj0 is the stack slot we're trying to
2630   //  get to.
2631   //
2632   //    ----------------------------------
2633   //    | BP | Obj0 | Obj1 | ... | ObjN |
2634   //    ----------------------------------
2635   //    ^    ^      ^                   ^
2636   //    A    B      C                   E
2637   //
2638   // A is the incoming stack pointer.
2639   // (B - A) is the local area offset (-8 for x86-64) [1]
2640   // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2]
2641   //
2642   // |(E - B)| is the StackSize (absolute value, positive).  For a
2643   // stack that grown down, this works out to be (B - E). [3]
2644   //
2645   // E is also the value of %rsp after stack has been set up, and we
2646   // want (C - E) -- the value we can add to %rsp to get to Obj0.  Now
2647   // (C - E) == (C - A) - (B - A) + (B - E)
2648   //            { Using [1], [2] and [3] above }
2649   //         == getObjectOffset - LocalAreaOffset + StackSize
2650 
2651   return getFrameIndexReferenceSP(MF, FI, FrameReg, StackSize);
2652 }
2653 
assignCalleeSavedSpillSlots(MachineFunction & MF,const TargetRegisterInfo * TRI,std::vector<CalleeSavedInfo> & CSI) const2654 bool X86FrameLowering::assignCalleeSavedSpillSlots(
2655     MachineFunction &MF, const TargetRegisterInfo *TRI,
2656     std::vector<CalleeSavedInfo> &CSI) const {
2657   MachineFrameInfo &MFI = MF.getFrameInfo();
2658   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2659 
2660   unsigned CalleeSavedFrameSize = 0;
2661   unsigned XMMCalleeSavedFrameSize = 0;
2662   auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2663   int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
2664 
2665   int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2666 
2667   if (TailCallReturnAddrDelta < 0) {
2668     // create RETURNADDR area
2669     //   arg
2670     //   arg
2671     //   RETADDR
2672     //   { ...
2673     //     RETADDR area
2674     //     ...
2675     //   }
2676     //   [EBP]
2677     MFI.CreateFixedObject(-TailCallReturnAddrDelta,
2678                            TailCallReturnAddrDelta - SlotSize, true);
2679   }
2680 
2681   // Spill the BasePtr if it's used.
2682   if (this->TRI->hasBasePointer(MF)) {
2683     // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
2684     if (MF.hasEHFunclets()) {
2685       int FI = MFI.CreateSpillStackObject(SlotSize, Align(SlotSize));
2686       X86FI->setHasSEHFramePtrSave(true);
2687       X86FI->setSEHFramePtrSaveIndex(FI);
2688     }
2689   }
2690 
2691   if (hasFP(MF)) {
2692     // emitPrologue always spills frame register the first thing.
2693     SpillSlotOffset -= SlotSize;
2694     MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2695 
2696     // The async context lives directly before the frame pointer, and we
2697     // allocate a second slot to preserve stack alignment.
2698     if (X86FI->hasSwiftAsyncContext()) {
2699       SpillSlotOffset -= SlotSize;
2700       MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2701       SpillSlotOffset -= SlotSize;
2702     }
2703 
2704     // Since emitPrologue and emitEpilogue will handle spilling and restoring of
2705     // the frame register, we can delete it from CSI list and not have to worry
2706     // about avoiding it later.
2707     Register FPReg = TRI->getFrameRegister(MF);
2708     for (unsigned i = 0; i < CSI.size(); ++i) {
2709       if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) {
2710         CSI.erase(CSI.begin() + i);
2711         break;
2712       }
2713     }
2714   }
2715 
2716   // Assign slots for GPRs. It increases frame size.
2717   for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
2718     Register Reg = I.getReg();
2719 
2720     if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
2721       continue;
2722 
2723     SpillSlotOffset -= SlotSize;
2724     CalleeSavedFrameSize += SlotSize;
2725 
2726     int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2727     I.setFrameIdx(SlotIndex);
2728   }
2729 
2730   X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
2731   MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
2732 
2733   // Assign slots for XMMs.
2734   for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
2735     Register Reg = I.getReg();
2736     if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
2737       continue;
2738 
2739     // If this is k-register make sure we lookup via the largest legal type.
2740     MVT VT = MVT::Other;
2741     if (X86::VK16RegClass.contains(Reg))
2742       VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2743 
2744     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2745     unsigned Size = TRI->getSpillSize(*RC);
2746     Align Alignment = TRI->getSpillAlign(*RC);
2747     // ensure alignment
2748     assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86");
2749     SpillSlotOffset = -alignTo(-SpillSlotOffset, Alignment);
2750 
2751     // spill into slot
2752     SpillSlotOffset -= Size;
2753     int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset);
2754     I.setFrameIdx(SlotIndex);
2755     MFI.ensureMaxAlignment(Alignment);
2756 
2757     // Save the start offset and size of XMM in stack frame for funclets.
2758     if (X86::VR128RegClass.contains(Reg)) {
2759       WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize;
2760       XMMCalleeSavedFrameSize += Size;
2761     }
2762   }
2763 
2764   return true;
2765 }
2766 
spillCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const2767 bool X86FrameLowering::spillCalleeSavedRegisters(
2768     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2769     ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2770   DebugLoc DL = MBB.findDebugLoc(MI);
2771 
2772   // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
2773   // for us, and there are no XMM CSRs on Win32.
2774   if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
2775     return true;
2776 
2777   // Push GPRs. It increases frame size.
2778   const MachineFunction &MF = *MBB.getParent();
2779   unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2780   for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
2781     Register Reg = I.getReg();
2782 
2783     if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
2784       continue;
2785 
2786     const MachineRegisterInfo &MRI = MF.getRegInfo();
2787     bool isLiveIn = MRI.isLiveIn(Reg);
2788     if (!isLiveIn)
2789       MBB.addLiveIn(Reg);
2790 
2791     // Decide whether we can add a kill flag to the use.
2792     bool CanKill = !isLiveIn;
2793     // Check if any subregister is live-in
2794     if (CanKill) {
2795       for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg) {
2796         if (MRI.isLiveIn(*AReg)) {
2797           CanKill = false;
2798           break;
2799         }
2800       }
2801     }
2802 
2803     // Do not set a kill flag on values that are also marked as live-in. This
2804     // happens with the @llvm-returnaddress intrinsic and with arguments
2805     // passed in callee saved registers.
2806     // Omitting the kill flags is conservatively correct even if the live-in
2807     // is not used after all.
2808     BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, getKillRegState(CanKill))
2809       .setMIFlag(MachineInstr::FrameSetup);
2810   }
2811 
2812   // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
2813   // It can be done by spilling XMMs to stack frame.
2814   for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
2815     Register Reg = I.getReg();
2816     if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
2817       continue;
2818 
2819     // If this is k-register make sure we lookup via the largest legal type.
2820     MVT VT = MVT::Other;
2821     if (X86::VK16RegClass.contains(Reg))
2822       VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2823 
2824     // Add the callee-saved register as live-in. It's killed at the spill.
2825     MBB.addLiveIn(Reg);
2826     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2827 
2828     TII.storeRegToStackSlot(MBB, MI, Reg, true, I.getFrameIdx(), RC, TRI,
2829                             Register());
2830     --MI;
2831     MI->setFlag(MachineInstr::FrameSetup);
2832     ++MI;
2833   }
2834 
2835   return true;
2836 }
2837 
emitCatchRetReturnValue(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineInstr * CatchRet) const2838 void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB,
2839                                                MachineBasicBlock::iterator MBBI,
2840                                                MachineInstr *CatchRet) const {
2841   // SEH shouldn't use catchret.
2842   assert(!isAsynchronousEHPersonality(classifyEHPersonality(
2843              MBB.getParent()->getFunction().getPersonalityFn())) &&
2844          "SEH should not use CATCHRET");
2845   const DebugLoc &DL = CatchRet->getDebugLoc();
2846   MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(0).getMBB();
2847 
2848   // Fill EAX/RAX with the address of the target block.
2849   if (STI.is64Bit()) {
2850     // LEA64r CatchRetTarget(%rip), %rax
2851     BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), X86::RAX)
2852         .addReg(X86::RIP)
2853         .addImm(0)
2854         .addReg(0)
2855         .addMBB(CatchRetTarget)
2856         .addReg(0);
2857   } else {
2858     // MOV32ri $CatchRetTarget, %eax
2859     BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
2860         .addMBB(CatchRetTarget);
2861   }
2862 
2863   // Record that we've taken the address of CatchRetTarget and no longer just
2864   // reference it in a terminator.
2865   CatchRetTarget->setMachineBlockAddressTaken();
2866 }
2867 
restoreCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,MutableArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const2868 bool X86FrameLowering::restoreCalleeSavedRegisters(
2869     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2870     MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2871   if (CSI.empty())
2872     return false;
2873 
2874   if (MI != MBB.end() && isFuncletReturnInstr(*MI) && STI.isOSWindows()) {
2875     // Don't restore CSRs in 32-bit EH funclets. Matches
2876     // spillCalleeSavedRegisters.
2877     if (STI.is32Bit())
2878       return true;
2879     // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
2880     // funclets. emitEpilogue transforms these to normal jumps.
2881     if (MI->getOpcode() == X86::CATCHRET) {
2882       const Function &F = MBB.getParent()->getFunction();
2883       bool IsSEH = isAsynchronousEHPersonality(
2884           classifyEHPersonality(F.getPersonalityFn()));
2885       if (IsSEH)
2886         return true;
2887     }
2888   }
2889 
2890   DebugLoc DL = MBB.findDebugLoc(MI);
2891 
2892   // Reload XMMs from stack frame.
2893   for (const CalleeSavedInfo &I : CSI) {
2894     Register Reg = I.getReg();
2895     if (X86::GR64RegClass.contains(Reg) ||
2896         X86::GR32RegClass.contains(Reg))
2897       continue;
2898 
2899     // If this is k-register make sure we lookup via the largest legal type.
2900     MVT VT = MVT::Other;
2901     if (X86::VK16RegClass.contains(Reg))
2902       VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2903 
2904     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2905     TII.loadRegFromStackSlot(MBB, MI, Reg, I.getFrameIdx(), RC, TRI,
2906                              Register());
2907   }
2908 
2909   // POP GPRs.
2910   unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
2911   for (const CalleeSavedInfo &I : CSI) {
2912     Register Reg = I.getReg();
2913     if (!X86::GR64RegClass.contains(Reg) &&
2914         !X86::GR32RegClass.contains(Reg))
2915       continue;
2916 
2917     BuildMI(MBB, MI, DL, TII.get(Opc), Reg)
2918         .setMIFlag(MachineInstr::FrameDestroy);
2919   }
2920   return true;
2921 }
2922 
determineCalleeSaves(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const2923 void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
2924                                             BitVector &SavedRegs,
2925                                             RegScavenger *RS) const {
2926   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2927 
2928   // Spill the BasePtr if it's used.
2929   if (TRI->hasBasePointer(MF)){
2930     Register BasePtr = TRI->getBaseRegister();
2931     if (STI.isTarget64BitILP32())
2932       BasePtr = getX86SubSuperRegister(BasePtr, 64);
2933     SavedRegs.set(BasePtr);
2934   }
2935 }
2936 
2937 static bool
HasNestArgument(const MachineFunction * MF)2938 HasNestArgument(const MachineFunction *MF) {
2939   const Function &F = MF->getFunction();
2940   for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end();
2941        I != E; I++) {
2942     if (I->hasNestAttr() && !I->use_empty())
2943       return true;
2944   }
2945   return false;
2946 }
2947 
2948 /// GetScratchRegister - Get a temp register for performing work in the
2949 /// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
2950 /// and the properties of the function either one or two registers will be
2951 /// needed. Set primary to true for the first register, false for the second.
2952 static unsigned
GetScratchRegister(bool Is64Bit,bool IsLP64,const MachineFunction & MF,bool Primary)2953 GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary) {
2954   CallingConv::ID CallingConvention = MF.getFunction().getCallingConv();
2955 
2956   // Erlang stuff.
2957   if (CallingConvention == CallingConv::HiPE) {
2958     if (Is64Bit)
2959       return Primary ? X86::R14 : X86::R13;
2960     else
2961       return Primary ? X86::EBX : X86::EDI;
2962   }
2963 
2964   if (Is64Bit) {
2965     if (IsLP64)
2966       return Primary ? X86::R11 : X86::R12;
2967     else
2968       return Primary ? X86::R11D : X86::R12D;
2969   }
2970 
2971   bool IsNested = HasNestArgument(&MF);
2972 
2973   if (CallingConvention == CallingConv::X86_FastCall ||
2974       CallingConvention == CallingConv::Fast ||
2975       CallingConvention == CallingConv::Tail) {
2976     if (IsNested)
2977       report_fatal_error("Segmented stacks does not support fastcall with "
2978                          "nested function.");
2979     return Primary ? X86::EAX : X86::ECX;
2980   }
2981   if (IsNested)
2982     return Primary ? X86::EDX : X86::EAX;
2983   return Primary ? X86::ECX : X86::EAX;
2984 }
2985 
2986 // The stack limit in the TCB is set to this many bytes above the actual stack
2987 // limit.
2988 static const uint64_t kSplitStackAvailable = 256;
2989 
adjustForSegmentedStacks(MachineFunction & MF,MachineBasicBlock & PrologueMBB) const2990 void X86FrameLowering::adjustForSegmentedStacks(
2991     MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
2992   MachineFrameInfo &MFI = MF.getFrameInfo();
2993   uint64_t StackSize;
2994   unsigned TlsReg, TlsOffset;
2995   DebugLoc DL;
2996 
2997   // To support shrink-wrapping we would need to insert the new blocks
2998   // at the right place and update the branches to PrologueMBB.
2999   assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3000 
3001   unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3002   assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3003          "Scratch register is live-in");
3004 
3005   if (MF.getFunction().isVarArg())
3006     report_fatal_error("Segmented stacks do not support vararg functions.");
3007   if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
3008       !STI.isTargetWin64() && !STI.isTargetFreeBSD() &&
3009       !STI.isTargetDragonFly())
3010     report_fatal_error("Segmented stacks not supported on this platform.");
3011 
3012   // Eventually StackSize will be calculated by a link-time pass; which will
3013   // also decide whether checking code needs to be injected into this particular
3014   // prologue.
3015   StackSize = MFI.getStackSize();
3016 
3017   if (!MFI.needsSplitStackProlog())
3018     return;
3019 
3020   MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
3021   MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
3022   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3023   bool IsNested = false;
3024 
3025   // We need to know if the function has a nest argument only in 64 bit mode.
3026   if (Is64Bit)
3027     IsNested = HasNestArgument(&MF);
3028 
3029   // The MOV R10, RAX needs to be in a different block, since the RET we emit in
3030   // allocMBB needs to be last (terminating) instruction.
3031 
3032   for (const auto &LI : PrologueMBB.liveins()) {
3033     allocMBB->addLiveIn(LI);
3034     checkMBB->addLiveIn(LI);
3035   }
3036 
3037   if (IsNested)
3038     allocMBB->addLiveIn(IsLP64 ? X86::R10 : X86::R10D);
3039 
3040   MF.push_front(allocMBB);
3041   MF.push_front(checkMBB);
3042 
3043   // When the frame size is less than 256 we just compare the stack
3044   // boundary directly to the value of the stack pointer, per gcc.
3045   bool CompareStackPointer = StackSize < kSplitStackAvailable;
3046 
3047   // Read the limit off the current stacklet off the stack_guard location.
3048   if (Is64Bit) {
3049     if (STI.isTargetLinux()) {
3050       TlsReg = X86::FS;
3051       TlsOffset = IsLP64 ? 0x70 : 0x40;
3052     } else if (STI.isTargetDarwin()) {
3053       TlsReg = X86::GS;
3054       TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90.
3055     } else if (STI.isTargetWin64()) {
3056       TlsReg = X86::GS;
3057       TlsOffset = 0x28; // pvArbitrary, reserved for application use
3058     } else if (STI.isTargetFreeBSD()) {
3059       TlsReg = X86::FS;
3060       TlsOffset = 0x18;
3061     } else if (STI.isTargetDragonFly()) {
3062       TlsReg = X86::FS;
3063       TlsOffset = 0x20; // use tls_tcb.tcb_segstack
3064     } else {
3065       report_fatal_error("Segmented stacks not supported on this platform.");
3066     }
3067 
3068     if (CompareStackPointer)
3069       ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
3070     else
3071       BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::LEA64r : X86::LEA64_32r), ScratchReg).addReg(X86::RSP)
3072         .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
3073 
3074     BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::CMP64rm : X86::CMP32rm)).addReg(ScratchReg)
3075       .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg);
3076   } else {
3077     if (STI.isTargetLinux()) {
3078       TlsReg = X86::GS;
3079       TlsOffset = 0x30;
3080     } else if (STI.isTargetDarwin()) {
3081       TlsReg = X86::GS;
3082       TlsOffset = 0x48 + 90*4;
3083     } else if (STI.isTargetWin32()) {
3084       TlsReg = X86::FS;
3085       TlsOffset = 0x14; // pvArbitrary, reserved for application use
3086     } else if (STI.isTargetDragonFly()) {
3087       TlsReg = X86::FS;
3088       TlsOffset = 0x10; // use tls_tcb.tcb_segstack
3089     } else if (STI.isTargetFreeBSD()) {
3090       report_fatal_error("Segmented stacks not supported on FreeBSD i386.");
3091     } else {
3092       report_fatal_error("Segmented stacks not supported on this platform.");
3093     }
3094 
3095     if (CompareStackPointer)
3096       ScratchReg = X86::ESP;
3097     else
3098       BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP)
3099         .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
3100 
3101     if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() ||
3102         STI.isTargetDragonFly()) {
3103       BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg)
3104         .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg);
3105     } else if (STI.isTargetDarwin()) {
3106 
3107       // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
3108       unsigned ScratchReg2;
3109       bool SaveScratch2;
3110       if (CompareStackPointer) {
3111         // The primary scratch register is available for holding the TLS offset.
3112         ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3113         SaveScratch2 = false;
3114       } else {
3115         // Need to use a second register to hold the TLS offset
3116         ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, false);
3117 
3118         // Unfortunately, with fastcc the second scratch register may hold an
3119         // argument.
3120         SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2);
3121       }
3122 
3123       // If Scratch2 is live-in then it needs to be saved.
3124       assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
3125              "Scratch register is live-in and not saved");
3126 
3127       if (SaveScratch2)
3128         BuildMI(checkMBB, DL, TII.get(X86::PUSH32r))
3129           .addReg(ScratchReg2, RegState::Kill);
3130 
3131       BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2)
3132         .addImm(TlsOffset);
3133       BuildMI(checkMBB, DL, TII.get(X86::CMP32rm))
3134         .addReg(ScratchReg)
3135         .addReg(ScratchReg2).addImm(1).addReg(0)
3136         .addImm(0)
3137         .addReg(TlsReg);
3138 
3139       if (SaveScratch2)
3140         BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2);
3141     }
3142   }
3143 
3144   // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
3145   // It jumps to normal execution of the function body.
3146   BuildMI(checkMBB, DL, TII.get(X86::JCC_1)).addMBB(&PrologueMBB).addImm(X86::COND_A);
3147 
3148   // On 32 bit we first push the arguments size and then the frame size. On 64
3149   // bit, we pass the stack frame size in r10 and the argument size in r11.
3150   if (Is64Bit) {
3151     // Functions with nested arguments use R10, so it needs to be saved across
3152     // the call to _morestack
3153 
3154     const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
3155     const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
3156     const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
3157     const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3158 
3159     if (IsNested)
3160       BuildMI(allocMBB, DL, TII.get(MOVrr), RegAX).addReg(Reg10);
3161 
3162     BuildMI(allocMBB, DL, TII.get(getMOVriOpcode(IsLP64, StackSize)), Reg10)
3163         .addImm(StackSize);
3164     BuildMI(allocMBB, DL,
3165             TII.get(getMOVriOpcode(IsLP64, X86FI->getArgumentStackSize())),
3166             Reg11)
3167         .addImm(X86FI->getArgumentStackSize());
3168   } else {
3169     BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
3170       .addImm(X86FI->getArgumentStackSize());
3171     BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
3172       .addImm(StackSize);
3173   }
3174 
3175   // __morestack is in libgcc
3176   if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
3177     // Under the large code model, we cannot assume that __morestack lives
3178     // within 2^31 bytes of the call site, so we cannot use pc-relative
3179     // addressing. We cannot perform the call via a temporary register,
3180     // as the rax register may be used to store the static chain, and all
3181     // other suitable registers may be either callee-save or used for
3182     // parameter passing. We cannot use the stack at this point either
3183     // because __morestack manipulates the stack directly.
3184     //
3185     // To avoid these issues, perform an indirect call via a read-only memory
3186     // location containing the address.
3187     //
3188     // This solution is not perfect, as it assumes that the .rodata section
3189     // is laid out within 2^31 bytes of each function body, but this seems
3190     // to be sufficient for JIT.
3191     // FIXME: Add retpoline support and remove the error here..
3192     if (STI.useIndirectThunkCalls())
3193       report_fatal_error("Emitting morestack calls on 64-bit with the large "
3194                          "code model and thunks not yet implemented.");
3195     BuildMI(allocMBB, DL, TII.get(X86::CALL64m))
3196         .addReg(X86::RIP)
3197         .addImm(0)
3198         .addReg(0)
3199         .addExternalSymbol("__morestack_addr")
3200         .addReg(0);
3201   } else {
3202     if (Is64Bit)
3203       BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32))
3204         .addExternalSymbol("__morestack");
3205     else
3206       BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32))
3207         .addExternalSymbol("__morestack");
3208   }
3209 
3210   if (IsNested)
3211     BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10));
3212   else
3213     BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET));
3214 
3215   allocMBB->addSuccessor(&PrologueMBB);
3216 
3217   checkMBB->addSuccessor(allocMBB, BranchProbability::getZero());
3218   checkMBB->addSuccessor(&PrologueMBB, BranchProbability::getOne());
3219 
3220 #ifdef EXPENSIVE_CHECKS
3221   MF.verify();
3222 #endif
3223 }
3224 
3225 /// Lookup an ERTS parameter in the !hipe.literals named metadata node.
3226 /// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets
3227 /// to fields it needs, through a named metadata node "hipe.literals" containing
3228 /// name-value pairs.
getHiPELiteral(NamedMDNode * HiPELiteralsMD,const StringRef LiteralName)3229 static unsigned getHiPELiteral(
3230     NamedMDNode *HiPELiteralsMD, const StringRef LiteralName) {
3231   for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) {
3232     MDNode *Node = HiPELiteralsMD->getOperand(i);
3233     if (Node->getNumOperands() != 2) continue;
3234     MDString *NodeName = dyn_cast<MDString>(Node->getOperand(0));
3235     ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Node->getOperand(1));
3236     if (!NodeName || !NodeVal) continue;
3237     ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(NodeVal->getValue());
3238     if (ValConst && NodeName->getString() == LiteralName) {
3239       return ValConst->getZExtValue();
3240     }
3241   }
3242 
3243   report_fatal_error("HiPE literal " + LiteralName
3244                      + " required but not provided");
3245 }
3246 
3247 // Return true if there are no non-ehpad successors to MBB and there are no
3248 // non-meta instructions between MBBI and MBB.end().
blockEndIsUnreachable(const MachineBasicBlock & MBB,MachineBasicBlock::const_iterator MBBI)3249 static bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
3250                                   MachineBasicBlock::const_iterator MBBI) {
3251   return llvm::all_of(
3252              MBB.successors(),
3253              [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
3254          std::all_of(MBBI, MBB.end(), [](const MachineInstr &MI) {
3255            return MI.isMetaInstruction();
3256          });
3257 }
3258 
3259 /// Erlang programs may need a special prologue to handle the stack size they
3260 /// might need at runtime. That is because Erlang/OTP does not implement a C
3261 /// stack but uses a custom implementation of hybrid stack/heap architecture.
3262 /// (for more information see Eric Stenman's Ph.D. thesis:
3263 /// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
3264 ///
3265 /// CheckStack:
3266 ///       temp0 = sp - MaxStack
3267 ///       if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3268 /// OldStart:
3269 ///       ...
3270 /// IncStack:
3271 ///       call inc_stack   # doubles the stack space
3272 ///       temp0 = sp - MaxStack
3273 ///       if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
adjustForHiPEPrologue(MachineFunction & MF,MachineBasicBlock & PrologueMBB) const3274 void X86FrameLowering::adjustForHiPEPrologue(
3275     MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3276   MachineFrameInfo &MFI = MF.getFrameInfo();
3277   DebugLoc DL;
3278 
3279   // To support shrink-wrapping we would need to insert the new blocks
3280   // at the right place and update the branches to PrologueMBB.
3281   assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3282 
3283   // HiPE-specific values
3284   NamedMDNode *HiPELiteralsMD = MF.getMMI().getModule()
3285     ->getNamedMetadata("hipe.literals");
3286   if (!HiPELiteralsMD)
3287     report_fatal_error(
3288         "Can't generate HiPE prologue without runtime parameters");
3289   const unsigned HipeLeafWords
3290     = getHiPELiteral(HiPELiteralsMD,
3291                      Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS");
3292   const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
3293   const unsigned Guaranteed = HipeLeafWords * SlotSize;
3294   unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs ?
3295                             MF.getFunction().arg_size() - CCRegisteredArgs : 0;
3296   unsigned MaxStack = MFI.getStackSize() + CallerStkArity*SlotSize + SlotSize;
3297 
3298   assert(STI.isTargetLinux() &&
3299          "HiPE prologue is only supported on Linux operating systems.");
3300 
3301   // Compute the largest caller's frame that is needed to fit the callees'
3302   // frames. This 'MaxStack' is computed from:
3303   //
3304   // a) the fixed frame size, which is the space needed for all spilled temps,
3305   // b) outgoing on-stack parameter areas, and
3306   // c) the minimum stack space this function needs to make available for the
3307   //    functions it calls (a tunable ABI property).
3308   if (MFI.hasCalls()) {
3309     unsigned MoreStackForCalls = 0;
3310 
3311     for (auto &MBB : MF) {
3312       for (auto &MI : MBB) {
3313         if (!MI.isCall())
3314           continue;
3315 
3316         // Get callee operand.
3317         const MachineOperand &MO = MI.getOperand(0);
3318 
3319         // Only take account of global function calls (no closures etc.).
3320         if (!MO.isGlobal())
3321           continue;
3322 
3323         const Function *F = dyn_cast<Function>(MO.getGlobal());
3324         if (!F)
3325           continue;
3326 
3327         // Do not update 'MaxStack' for primitive and built-in functions
3328         // (encoded with names either starting with "erlang."/"bif_" or not
3329         // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
3330         // "_", such as the BIF "suspend_0") as they are executed on another
3331         // stack.
3332         if (F->getName().contains("erlang.") || F->getName().contains("bif_") ||
3333             F->getName().find_first_of("._") == StringRef::npos)
3334           continue;
3335 
3336         unsigned CalleeStkArity =
3337           F->arg_size() > CCRegisteredArgs ? F->arg_size()-CCRegisteredArgs : 0;
3338         if (HipeLeafWords - 1 > CalleeStkArity)
3339           MoreStackForCalls = std::max(MoreStackForCalls,
3340                                (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
3341       }
3342     }
3343     MaxStack += MoreStackForCalls;
3344   }
3345 
3346   // If the stack frame needed is larger than the guaranteed then runtime checks
3347   // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
3348   if (MaxStack > Guaranteed) {
3349     MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
3350     MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
3351 
3352     for (const auto &LI : PrologueMBB.liveins()) {
3353       stackCheckMBB->addLiveIn(LI);
3354       incStackMBB->addLiveIn(LI);
3355     }
3356 
3357     MF.push_front(incStackMBB);
3358     MF.push_front(stackCheckMBB);
3359 
3360     unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3361     unsigned LEAop, CMPop, CALLop;
3362     SPLimitOffset = getHiPELiteral(HiPELiteralsMD, "P_NSP_LIMIT");
3363     if (Is64Bit) {
3364       SPReg = X86::RSP;
3365       PReg  = X86::RBP;
3366       LEAop = X86::LEA64r;
3367       CMPop = X86::CMP64rm;
3368       CALLop = X86::CALL64pcrel32;
3369     } else {
3370       SPReg = X86::ESP;
3371       PReg  = X86::EBP;
3372       LEAop = X86::LEA32r;
3373       CMPop = X86::CMP32rm;
3374       CALLop = X86::CALLpcrel32;
3375     }
3376 
3377     ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3378     assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3379            "HiPE prologue scratch register is live-in");
3380 
3381     // Create new MBB for StackCheck:
3382     addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg),
3383                  SPReg, false, -MaxStack);
3384     // SPLimitOffset is in a fixed heap location (pointed by BP).
3385     addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop))
3386                  .addReg(ScratchReg), PReg, false, SPLimitOffset);
3387     BuildMI(stackCheckMBB, DL, TII.get(X86::JCC_1)).addMBB(&PrologueMBB).addImm(X86::COND_AE);
3388 
3389     // Create new MBB for IncStack:
3390     BuildMI(incStackMBB, DL, TII.get(CALLop)).
3391       addExternalSymbol("inc_stack_0");
3392     addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg),
3393                  SPReg, false, -MaxStack);
3394     addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop))
3395                  .addReg(ScratchReg), PReg, false, SPLimitOffset);
3396     BuildMI(incStackMBB, DL, TII.get(X86::JCC_1)).addMBB(incStackMBB).addImm(X86::COND_LE);
3397 
3398     stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100});
3399     stackCheckMBB->addSuccessor(incStackMBB, {1, 100});
3400     incStackMBB->addSuccessor(&PrologueMBB, {99, 100});
3401     incStackMBB->addSuccessor(incStackMBB, {1, 100});
3402   }
3403 #ifdef EXPENSIVE_CHECKS
3404   MF.verify();
3405 #endif
3406 }
3407 
adjustStackWithPops(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,int Offset) const3408 bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
3409                                            MachineBasicBlock::iterator MBBI,
3410                                            const DebugLoc &DL,
3411                                            int Offset) const {
3412   if (Offset <= 0)
3413     return false;
3414 
3415   if (Offset % SlotSize)
3416     return false;
3417 
3418   int NumPops = Offset / SlotSize;
3419   // This is only worth it if we have at most 2 pops.
3420   if (NumPops != 1 && NumPops != 2)
3421     return false;
3422 
3423   // Handle only the trivial case where the adjustment directly follows
3424   // a call. This is the most common one, anyway.
3425   if (MBBI == MBB.begin())
3426     return false;
3427   MachineBasicBlock::iterator Prev = std::prev(MBBI);
3428   if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
3429     return false;
3430 
3431   unsigned Regs[2];
3432   unsigned FoundRegs = 0;
3433 
3434   const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3435   const MachineOperand &RegMask = Prev->getOperand(1);
3436 
3437   auto &RegClass =
3438       Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3439   // Try to find up to NumPops free registers.
3440   for (auto Candidate : RegClass) {
3441     // Poor man's liveness:
3442     // Since we're immediately after a call, any register that is clobbered
3443     // by the call and not defined by it can be considered dead.
3444     if (!RegMask.clobbersPhysReg(Candidate))
3445       continue;
3446 
3447     // Don't clobber reserved registers
3448     if (MRI.isReserved(Candidate))
3449       continue;
3450 
3451     bool IsDef = false;
3452     for (const MachineOperand &MO : Prev->implicit_operands()) {
3453       if (MO.isReg() && MO.isDef() &&
3454           TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) {
3455         IsDef = true;
3456         break;
3457       }
3458     }
3459 
3460     if (IsDef)
3461       continue;
3462 
3463     Regs[FoundRegs++] = Candidate;
3464     if (FoundRegs == (unsigned)NumPops)
3465       break;
3466   }
3467 
3468   if (FoundRegs == 0)
3469     return false;
3470 
3471   // If we found only one free register, but need two, reuse the same one twice.
3472   while (FoundRegs < (unsigned)NumPops)
3473     Regs[FoundRegs++] = Regs[0];
3474 
3475   for (int i = 0; i < NumPops; ++i)
3476     BuildMI(MBB, MBBI, DL,
3477             TII.get(STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]);
3478 
3479   return true;
3480 }
3481 
3482 MachineBasicBlock::iterator X86FrameLowering::
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I) const3483 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
3484                               MachineBasicBlock::iterator I) const {
3485   bool reserveCallFrame = hasReservedCallFrame(MF);
3486   unsigned Opcode = I->getOpcode();
3487   bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
3488   DebugLoc DL = I->getDebugLoc(); // copy DebugLoc as I will be erased.
3489   uint64_t Amount = TII.getFrameSize(*I);
3490   uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(*I) : 0;
3491   I = MBB.erase(I);
3492   auto InsertPos = skipDebugInstructionsForward(I, MBB.end());
3493 
3494   // Try to avoid emitting dead SP adjustments if the block end is unreachable,
3495   // typically because the function is marked noreturn (abort, throw,
3496   // assert_fail, etc).
3497   if (isDestroy && blockEndIsUnreachable(MBB, I))
3498     return I;
3499 
3500   if (!reserveCallFrame) {
3501     // If the stack pointer can be changed after prologue, turn the
3502     // adjcallstackup instruction into a 'sub ESP, <amt>' and the
3503     // adjcallstackdown instruction into 'add ESP, <amt>'
3504 
3505     // We need to keep the stack aligned properly.  To do this, we round the
3506     // amount of space needed for the outgoing arguments up to the next
3507     // alignment boundary.
3508     Amount = alignTo(Amount, getStackAlign());
3509 
3510     const Function &F = MF.getFunction();
3511     bool WindowsCFI = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
3512     bool DwarfCFI = !WindowsCFI && MF.needsFrameMoves();
3513 
3514     // If we have any exception handlers in this function, and we adjust
3515     // the SP before calls, we may need to indicate this to the unwinder
3516     // using GNU_ARGS_SIZE. Note that this may be necessary even when
3517     // Amount == 0, because the preceding function may have set a non-0
3518     // GNU_ARGS_SIZE.
3519     // TODO: We don't need to reset this between subsequent functions,
3520     // if it didn't change.
3521     bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty();
3522 
3523     if (HasDwarfEHHandlers && !isDestroy &&
3524         MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences())
3525       BuildCFI(MBB, InsertPos, DL,
3526                MCCFIInstruction::createGnuArgsSize(nullptr, Amount));
3527 
3528     if (Amount == 0)
3529       return I;
3530 
3531     // Factor out the amount that gets handled inside the sequence
3532     // (Pushes of argument for frame setup, callee pops for frame destroy)
3533     Amount -= InternalAmt;
3534 
3535     // TODO: This is needed only if we require precise CFA.
3536     // If this is a callee-pop calling convention, emit a CFA adjust for
3537     // the amount the callee popped.
3538     if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF))
3539       BuildCFI(MBB, InsertPos, DL,
3540                MCCFIInstruction::createAdjustCfaOffset(nullptr, -InternalAmt));
3541 
3542     // Add Amount to SP to destroy a frame, or subtract to setup.
3543     int64_t StackAdjustment = isDestroy ? Amount : -Amount;
3544 
3545     if (StackAdjustment) {
3546       // Merge with any previous or following adjustment instruction. Note: the
3547       // instructions merged with here do not have CFI, so their stack
3548       // adjustments do not feed into CfaAdjustment.
3549       StackAdjustment += mergeSPUpdates(MBB, InsertPos, true);
3550       StackAdjustment += mergeSPUpdates(MBB, InsertPos, false);
3551 
3552       if (StackAdjustment) {
3553         if (!(F.hasMinSize() &&
3554               adjustStackWithPops(MBB, InsertPos, DL, StackAdjustment)))
3555           BuildStackAdjustment(MBB, InsertPos, DL, StackAdjustment,
3556                                /*InEpilogue=*/false);
3557       }
3558     }
3559 
3560     if (DwarfCFI && !hasFP(MF)) {
3561       // If we don't have FP, but need to generate unwind information,
3562       // we need to set the correct CFA offset after the stack adjustment.
3563       // How much we adjust the CFA offset depends on whether we're emitting
3564       // CFI only for EH purposes or for debugging. EH only requires the CFA
3565       // offset to be correct at each call site, while for debugging we want
3566       // it to be more precise.
3567 
3568       int64_t CfaAdjustment = -StackAdjustment;
3569       // TODO: When not using precise CFA, we also need to adjust for the
3570       // InternalAmt here.
3571       if (CfaAdjustment) {
3572         BuildCFI(MBB, InsertPos, DL,
3573                  MCCFIInstruction::createAdjustCfaOffset(nullptr,
3574                                                          CfaAdjustment));
3575       }
3576     }
3577 
3578     return I;
3579   }
3580 
3581   if (InternalAmt) {
3582     MachineBasicBlock::iterator CI = I;
3583     MachineBasicBlock::iterator B = MBB.begin();
3584     while (CI != B && !std::prev(CI)->isCall())
3585       --CI;
3586     BuildStackAdjustment(MBB, CI, DL, -InternalAmt, /*InEpilogue=*/false);
3587   }
3588 
3589   return I;
3590 }
3591 
canUseAsPrologue(const MachineBasicBlock & MBB) const3592 bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
3593   assert(MBB.getParent() && "Block is not attached to a function!");
3594   const MachineFunction &MF = *MBB.getParent();
3595   if (!MBB.isLiveIn(X86::EFLAGS))
3596     return true;
3597 
3598   // If stack probes have to loop inline or call, that will clobber EFLAGS.
3599   // FIXME: we could allow cases that will use emitStackProbeInlineGenericBlock.
3600   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
3601   const X86TargetLowering &TLI = *STI.getTargetLowering();
3602   if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
3603     return false;
3604 
3605   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3606   return !TRI->hasStackRealignment(MF) && !X86FI->hasSwiftAsyncContext();
3607 }
3608 
canUseAsEpilogue(const MachineBasicBlock & MBB) const3609 bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
3610   assert(MBB.getParent() && "Block is not attached to a function!");
3611 
3612   // Win64 has strict requirements in terms of epilogue and we are
3613   // not taking a chance at messing with them.
3614   // I.e., unless this block is already an exit block, we can't use
3615   // it as an epilogue.
3616   if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
3617     return false;
3618 
3619   // Swift async context epilogue has a BTR instruction that clobbers parts of
3620   // EFLAGS.
3621   const MachineFunction &MF = *MBB.getParent();
3622   if (MF.getInfo<X86MachineFunctionInfo>()->hasSwiftAsyncContext())
3623     return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
3624 
3625   if (canUseLEAForSPInEpilogue(*MBB.getParent()))
3626     return true;
3627 
3628   // If we cannot use LEA to adjust SP, we may need to use ADD, which
3629   // clobbers the EFLAGS. Check that we do not need to preserve it,
3630   // otherwise, conservatively assume this is not
3631   // safe to insert the epilogue here.
3632   return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
3633 }
3634 
enableShrinkWrapping(const MachineFunction & MF) const3635 bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
3636   // If we may need to emit frameless compact unwind information, give
3637   // up as this is currently broken: PR25614.
3638   bool CompactUnwind =
3639       MF.getMMI().getContext().getObjectFileInfo()->getCompactUnwindSection() !=
3640       nullptr;
3641   return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF) ||
3642           !CompactUnwind) &&
3643          // The lowering of segmented stack and HiPE only support entry
3644          // blocks as prologue blocks: PR26107. This limitation may be
3645          // lifted if we fix:
3646          // - adjustForSegmentedStacks
3647          // - adjustForHiPEPrologue
3648          MF.getFunction().getCallingConv() != CallingConv::HiPE &&
3649          !MF.shouldSplitStack();
3650 }
3651 
restoreWin32EHStackPointers(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool RestoreSP) const3652 MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
3653     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
3654     const DebugLoc &DL, bool RestoreSP) const {
3655   assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
3656   assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
3657   assert(STI.is32Bit() && !Uses64BitFramePtr &&
3658          "restoring EBP/ESI on non-32-bit target");
3659 
3660   MachineFunction &MF = *MBB.getParent();
3661   Register FramePtr = TRI->getFrameRegister(MF);
3662   Register BasePtr = TRI->getBaseRegister();
3663   WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
3664   X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3665   MachineFrameInfo &MFI = MF.getFrameInfo();
3666 
3667   // FIXME: Don't set FrameSetup flag in catchret case.
3668 
3669   int FI = FuncInfo.EHRegNodeFrameIndex;
3670   int EHRegSize = MFI.getObjectSize(FI);
3671 
3672   if (RestoreSP) {
3673     // MOV32rm -EHRegSize(%ebp), %esp
3674     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), X86::ESP),
3675                  X86::EBP, true, -EHRegSize)
3676         .setMIFlag(MachineInstr::FrameSetup);
3677   }
3678 
3679   Register UsedReg;
3680   int EHRegOffset = getFrameIndexReference(MF, FI, UsedReg).getFixed();
3681   int EndOffset = -EHRegOffset - EHRegSize;
3682   FuncInfo.EHRegNodeEndOffset = EndOffset;
3683 
3684   if (UsedReg == FramePtr) {
3685     // ADD $offset, %ebp
3686     unsigned ADDri = getADDriOpcode(false, EndOffset);
3687     BuildMI(MBB, MBBI, DL, TII.get(ADDri), FramePtr)
3688         .addReg(FramePtr)
3689         .addImm(EndOffset)
3690         .setMIFlag(MachineInstr::FrameSetup)
3691         ->getOperand(3)
3692         .setIsDead();
3693     assert(EndOffset >= 0 &&
3694            "end of registration object above normal EBP position!");
3695   } else if (UsedReg == BasePtr) {
3696     // LEA offset(%ebp), %esi
3697     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA32r), BasePtr),
3698                  FramePtr, false, EndOffset)
3699         .setMIFlag(MachineInstr::FrameSetup);
3700     // MOV32rm SavedEBPOffset(%esi), %ebp
3701     assert(X86FI->getHasSEHFramePtrSave());
3702     int Offset =
3703         getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg)
3704             .getFixed();
3705     assert(UsedReg == BasePtr);
3706     addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), FramePtr),
3707                  UsedReg, true, Offset)
3708         .setMIFlag(MachineInstr::FrameSetup);
3709   } else {
3710     llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
3711   }
3712   return MBBI;
3713 }
3714 
getInitialCFAOffset(const MachineFunction & MF) const3715 int X86FrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
3716   return TRI->getSlotSize();
3717 }
3718 
3719 Register
getInitialCFARegister(const MachineFunction & MF) const3720 X86FrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
3721   return TRI->getDwarfRegNum(StackPtr, true);
3722 }
3723 
3724 namespace {
3725 // Struct used by orderFrameObjects to help sort the stack objects.
3726 struct X86FrameSortingObject {
3727   bool IsValid = false;         // true if we care about this Object.
3728   unsigned ObjectIndex = 0;     // Index of Object into MFI list.
3729   unsigned ObjectSize = 0;      // Size of Object in bytes.
3730   Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
3731   unsigned ObjectNumUses = 0;   // Object static number of uses.
3732 };
3733 
3734 // The comparison function we use for std::sort to order our local
3735 // stack symbols. The current algorithm is to use an estimated
3736 // "density". This takes into consideration the size and number of
3737 // uses each object has in order to roughly minimize code size.
3738 // So, for example, an object of size 16B that is referenced 5 times
3739 // will get higher priority than 4 4B objects referenced 1 time each.
3740 // It's not perfect and we may be able to squeeze a few more bytes out of
3741 // it (for example : 0(esp) requires fewer bytes, symbols allocated at the
3742 // fringe end can have special consideration, given their size is less
3743 // important, etc.), but the algorithmic complexity grows too much to be
3744 // worth the extra gains we get. This gets us pretty close.
3745 // The final order leaves us with objects with highest priority going
3746 // at the end of our list.
3747 struct X86FrameSortingComparator {
operator ()__anonffc28aca0411::X86FrameSortingComparator3748   inline bool operator()(const X86FrameSortingObject &A,
3749                          const X86FrameSortingObject &B) const {
3750     uint64_t DensityAScaled, DensityBScaled;
3751 
3752     // For consistency in our comparison, all invalid objects are placed
3753     // at the end. This also allows us to stop walking when we hit the
3754     // first invalid item after it's all sorted.
3755     if (!A.IsValid)
3756       return false;
3757     if (!B.IsValid)
3758       return true;
3759 
3760     // The density is calculated by doing :
3761     //     (double)DensityA = A.ObjectNumUses / A.ObjectSize
3762     //     (double)DensityB = B.ObjectNumUses / B.ObjectSize
3763     // Since this approach may cause inconsistencies in
3764     // the floating point <, >, == comparisons, depending on the floating
3765     // point model with which the compiler was built, we're going
3766     // to scale both sides by multiplying with
3767     // A.ObjectSize * B.ObjectSize. This ends up factoring away
3768     // the division and, with it, the need for any floating point
3769     // arithmetic.
3770     DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) *
3771       static_cast<uint64_t>(B.ObjectSize);
3772     DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) *
3773       static_cast<uint64_t>(A.ObjectSize);
3774 
3775     // If the two densities are equal, prioritize highest alignment
3776     // objects. This allows for similar alignment objects
3777     // to be packed together (given the same density).
3778     // There's room for improvement here, also, since we can pack
3779     // similar alignment (different density) objects next to each
3780     // other to save padding. This will also require further
3781     // complexity/iterations, and the overall gain isn't worth it,
3782     // in general. Something to keep in mind, though.
3783     if (DensityAScaled == DensityBScaled)
3784       return A.ObjectAlignment < B.ObjectAlignment;
3785 
3786     return DensityAScaled < DensityBScaled;
3787   }
3788 };
3789 } // namespace
3790 
3791 // Order the symbols in the local stack.
3792 // We want to place the local stack objects in some sort of sensible order.
3793 // The heuristic we use is to try and pack them according to static number
3794 // of uses and size of object in order to minimize code size.
orderFrameObjects(const MachineFunction & MF,SmallVectorImpl<int> & ObjectsToAllocate) const3795 void X86FrameLowering::orderFrameObjects(
3796     const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
3797   const MachineFrameInfo &MFI = MF.getFrameInfo();
3798 
3799   // Don't waste time if there's nothing to do.
3800   if (ObjectsToAllocate.empty())
3801     return;
3802 
3803   // Create an array of all MFI objects. We won't need all of these
3804   // objects, but we're going to create a full array of them to make
3805   // it easier to index into when we're counting "uses" down below.
3806   // We want to be able to easily/cheaply access an object by simply
3807   // indexing into it, instead of having to search for it every time.
3808   std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd());
3809 
3810   // Walk the objects we care about and mark them as such in our working
3811   // struct.
3812   for (auto &Obj : ObjectsToAllocate) {
3813     SortingObjects[Obj].IsValid = true;
3814     SortingObjects[Obj].ObjectIndex = Obj;
3815     SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(Obj);
3816     // Set the size.
3817     int ObjectSize = MFI.getObjectSize(Obj);
3818     if (ObjectSize == 0)
3819       // Variable size. Just use 4.
3820       SortingObjects[Obj].ObjectSize = 4;
3821     else
3822       SortingObjects[Obj].ObjectSize = ObjectSize;
3823   }
3824 
3825   // Count the number of uses for each object.
3826   for (auto &MBB : MF) {
3827     for (auto &MI : MBB) {
3828       if (MI.isDebugInstr())
3829         continue;
3830       for (const MachineOperand &MO : MI.operands()) {
3831         // Check to see if it's a local stack symbol.
3832         if (!MO.isFI())
3833           continue;
3834         int Index = MO.getIndex();
3835         // Check to see if it falls within our range, and is tagged
3836         // to require ordering.
3837         if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
3838             SortingObjects[Index].IsValid)
3839           SortingObjects[Index].ObjectNumUses++;
3840       }
3841     }
3842   }
3843 
3844   // Sort the objects using X86FrameSortingAlgorithm (see its comment for
3845   // info).
3846   llvm::stable_sort(SortingObjects, X86FrameSortingComparator());
3847 
3848   // Now modify the original list to represent the final order that
3849   // we want. The order will depend on whether we're going to access them
3850   // from the stack pointer or the frame pointer. For SP, the list should
3851   // end up with the END containing objects that we want with smaller offsets.
3852   // For FP, it should be flipped.
3853   int i = 0;
3854   for (auto &Obj : SortingObjects) {
3855     // All invalid items are sorted at the end, so it's safe to stop.
3856     if (!Obj.IsValid)
3857       break;
3858     ObjectsToAllocate[i++] = Obj.ObjectIndex;
3859   }
3860 
3861   // Flip it if we're accessing off of the FP.
3862   if (!TRI->hasStackRealignment(MF) && hasFP(MF))
3863     std::reverse(ObjectsToAllocate.begin(), ObjectsToAllocate.end());
3864 }
3865 
3866 
getWinEHParentFrameOffset(const MachineFunction & MF) const3867 unsigned X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const {
3868   // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
3869   unsigned Offset = 16;
3870   // RBP is immediately pushed.
3871   Offset += SlotSize;
3872   // All callee-saved registers are then pushed.
3873   Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
3874   // Every funclet allocates enough stack space for the largest outgoing call.
3875   Offset += getWinEHFuncletFrameSize(MF);
3876   return Offset;
3877 }
3878 
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const3879 void X86FrameLowering::processFunctionBeforeFrameFinalized(
3880     MachineFunction &MF, RegScavenger *RS) const {
3881   // Mark the function as not having WinCFI. We will set it back to true in
3882   // emitPrologue if it gets called and emits CFI.
3883   MF.setHasWinCFI(false);
3884 
3885   // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
3886   // aligned. The format doesn't support misaligned stack adjustments.
3887   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
3888     MF.getFrameInfo().ensureMaxAlignment(Align(SlotSize));
3889 
3890   // If this function isn't doing Win64-style C++ EH, we don't need to do
3891   // anything.
3892   if (STI.is64Bit() && MF.hasEHFunclets() &&
3893       classifyEHPersonality(MF.getFunction().getPersonalityFn()) ==
3894           EHPersonality::MSVC_CXX) {
3895     adjustFrameForMsvcCxxEh(MF);
3896   }
3897 }
3898 
adjustFrameForMsvcCxxEh(MachineFunction & MF) const3899 void X86FrameLowering::adjustFrameForMsvcCxxEh(MachineFunction &MF) const {
3900   // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
3901   // relative to RSP after the prologue.  Find the offset of the last fixed
3902   // object, so that we can allocate a slot immediately following it. If there
3903   // were no fixed objects, use offset -SlotSize, which is immediately after the
3904   // return address. Fixed objects have negative frame indices.
3905   MachineFrameInfo &MFI = MF.getFrameInfo();
3906   WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
3907   int64_t MinFixedObjOffset = -SlotSize;
3908   for (int I = MFI.getObjectIndexBegin(); I < 0; ++I)
3909     MinFixedObjOffset = std::min(MinFixedObjOffset, MFI.getObjectOffset(I));
3910 
3911   for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
3912     for (WinEHHandlerType &H : TBME.HandlerArray) {
3913       int FrameIndex = H.CatchObj.FrameIndex;
3914       if (FrameIndex != INT_MAX) {
3915         // Ensure alignment.
3916         unsigned Align = MFI.getObjectAlign(FrameIndex).value();
3917         MinFixedObjOffset -= std::abs(MinFixedObjOffset) % Align;
3918         MinFixedObjOffset -= MFI.getObjectSize(FrameIndex);
3919         MFI.setObjectOffset(FrameIndex, MinFixedObjOffset);
3920       }
3921     }
3922   }
3923 
3924   // Ensure alignment.
3925   MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8;
3926   int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
3927   int UnwindHelpFI =
3928       MFI.CreateFixedObject(SlotSize, UnwindHelpOffset, /*IsImmutable=*/false);
3929   EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3930 
3931   // Store -2 into UnwindHelp on function entry. We have to scan forwards past
3932   // other frame setup instructions.
3933   MachineBasicBlock &MBB = MF.front();
3934   auto MBBI = MBB.begin();
3935   while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
3936     ++MBBI;
3937 
3938   DebugLoc DL = MBB.findDebugLoc(MBBI);
3939   addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mi32)),
3940                     UnwindHelpFI)
3941       .addImm(-2);
3942 }
3943 
getReturnProtector() const3944 const ReturnProtectorLowering *X86FrameLowering::getReturnProtector() const {
3945   return &RPL;
3946 }
3947 
processFunctionBeforeFrameIndicesReplaced(MachineFunction & MF,RegScavenger * RS) const3948 void X86FrameLowering::processFunctionBeforeFrameIndicesReplaced(
3949     MachineFunction &MF, RegScavenger *RS) const {
3950   if (STI.is32Bit() && MF.hasEHFunclets())
3951     restoreWinEHStackPointersInParent(MF);
3952 }
3953 
restoreWinEHStackPointersInParent(MachineFunction & MF) const3954 void X86FrameLowering::restoreWinEHStackPointersInParent(
3955     MachineFunction &MF) const {
3956   // 32-bit functions have to restore stack pointers when control is transferred
3957   // back to the parent function. These blocks are identified as eh pads that
3958   // are not funclet entries.
3959   bool IsSEH = isAsynchronousEHPersonality(
3960       classifyEHPersonality(MF.getFunction().getPersonalityFn()));
3961   for (MachineBasicBlock &MBB : MF) {
3962     bool NeedsRestore = MBB.isEHPad() && !MBB.isEHFuncletEntry();
3963     if (NeedsRestore)
3964       restoreWin32EHStackPointers(MBB, MBB.begin(), DebugLoc(),
3965                                   /*RestoreSP=*/IsSEH);
3966   }
3967 }
3968