1 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the X86 implementation of TargetFrameLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "X86FrameLowering.h"
14 #include "MCTargetDesc/X86MCTargetDesc.h"
15 #include "X86InstrBuilder.h"
16 #include "X86InstrInfo.h"
17 #include "X86MachineFunctionInfo.h"
18 #include "X86ReturnProtectorLowering.h"
19 #include "X86Subtarget.h"
20 #include "X86TargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/Analysis/EHPersonalities.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineFrameInfo.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/MachineModuleInfo.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/CodeGen/WinEHFuncInfo.h"
31 #include "llvm/IR/DataLayout.h"
32 #include "llvm/IR/Function.h"
33 #include "llvm/MC/MCAsmInfo.h"
34 #include "llvm/MC/MCObjectFileInfo.h"
35 #include "llvm/MC/MCSymbol.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Target/TargetOptions.h"
38 #include <cstdlib>
39
40 #define DEBUG_TYPE "x86-fl"
41
42 STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue");
43 STATISTIC(NumFrameExtraProbe,
44 "Number of extra stack probes generated in prologue");
45
46 using namespace llvm;
47
X86FrameLowering(const X86Subtarget & STI,MaybeAlign StackAlignOverride)48 X86FrameLowering::X86FrameLowering(const X86Subtarget &STI,
49 MaybeAlign StackAlignOverride)
50 : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(),
51 STI.is64Bit() ? -8 : -4),
52 STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()), RPL() {
53 // Cache a bunch of frame-related predicates for this subtarget.
54 SlotSize = TRI->getSlotSize();
55 Is64Bit = STI.is64Bit();
56 IsLP64 = STI.isTarget64BitLP64();
57 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
58 Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64();
59 StackPtr = TRI->getStackRegister();
60 SaveArgs = Is64Bit ? STI.getSaveArgs() : 0;
61 }
62
hasReservedCallFrame(const MachineFunction & MF) const63 bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
64 return !MF.getFrameInfo().hasVarSizedObjects() &&
65 !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() &&
66 !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall();
67 }
68
69 /// canSimplifyCallFramePseudos - If there is a reserved call frame, the
70 /// call frame pseudos can be simplified. Having a FP, as in the default
71 /// implementation, is not sufficient here since we can't always use it.
72 /// Use a more nuanced condition.
73 bool
canSimplifyCallFramePseudos(const MachineFunction & MF) const74 X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const {
75 return hasReservedCallFrame(MF) ||
76 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
77 (hasFP(MF) && !TRI->hasStackRealignment(MF)) ||
78 TRI->hasBasePointer(MF);
79 }
80
81 // needsFrameIndexResolution - Do we need to perform FI resolution for
82 // this function. Normally, this is required only when the function
83 // has any stack objects. However, FI resolution actually has another job,
84 // not apparent from the title - it resolves callframesetup/destroy
85 // that were not simplified earlier.
86 // So, this is required for x86 functions that have push sequences even
87 // when there are no stack objects.
88 bool
needsFrameIndexResolution(const MachineFunction & MF) const89 X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const {
90 return MF.getFrameInfo().hasStackObjects() ||
91 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences();
92 }
93
94 /// hasFP - Return true if the specified function should have a dedicated frame
95 /// pointer register. This is true if the function has variable sized allocas
96 /// or if frame pointer elimination is disabled.
hasFP(const MachineFunction & MF) const97 bool X86FrameLowering::hasFP(const MachineFunction &MF) const {
98 const MachineFrameInfo &MFI = MF.getFrameInfo();
99 return (MF.getTarget().Options.DisableFramePointerElim(MF) ||
100 TRI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
101 MFI.isFrameAddressTaken() || MFI.hasOpaqueSPAdjustment() ||
102 MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() ||
103 MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() ||
104 MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() ||
105 MFI.hasStackMap() || MFI.hasPatchPoint() ||
106 (isWin64Prologue(MF) && MFI.hasCopyImplyingStackAdjustment()) ||
107 SaveArgs);
108 }
109
getSUBriOpcode(bool IsLP64,int64_t Imm)110 static unsigned getSUBriOpcode(bool IsLP64, int64_t Imm) {
111 if (IsLP64) {
112 if (isInt<8>(Imm))
113 return X86::SUB64ri8;
114 return X86::SUB64ri32;
115 } else {
116 if (isInt<8>(Imm))
117 return X86::SUB32ri8;
118 return X86::SUB32ri;
119 }
120 }
121
getADDriOpcode(bool IsLP64,int64_t Imm)122 static unsigned getADDriOpcode(bool IsLP64, int64_t Imm) {
123 if (IsLP64) {
124 if (isInt<8>(Imm))
125 return X86::ADD64ri8;
126 return X86::ADD64ri32;
127 } else {
128 if (isInt<8>(Imm))
129 return X86::ADD32ri8;
130 return X86::ADD32ri;
131 }
132 }
133
getSUBrrOpcode(bool IsLP64)134 static unsigned getSUBrrOpcode(bool IsLP64) {
135 return IsLP64 ? X86::SUB64rr : X86::SUB32rr;
136 }
137
getADDrrOpcode(bool IsLP64)138 static unsigned getADDrrOpcode(bool IsLP64) {
139 return IsLP64 ? X86::ADD64rr : X86::ADD32rr;
140 }
141
getANDriOpcode(bool IsLP64,int64_t Imm)142 static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) {
143 if (IsLP64) {
144 if (isInt<8>(Imm))
145 return X86::AND64ri8;
146 return X86::AND64ri32;
147 }
148 if (isInt<8>(Imm))
149 return X86::AND32ri8;
150 return X86::AND32ri;
151 }
152
getLEArOpcode(bool IsLP64)153 static unsigned getLEArOpcode(bool IsLP64) {
154 return IsLP64 ? X86::LEA64r : X86::LEA32r;
155 }
156
getMOVriOpcode(bool Use64BitReg,int64_t Imm)157 static unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm) {
158 if (Use64BitReg) {
159 if (isUInt<32>(Imm))
160 return X86::MOV32ri64;
161 if (isInt<32>(Imm))
162 return X86::MOV64ri32;
163 return X86::MOV64ri;
164 }
165 return X86::MOV32ri;
166 }
167
isEAXLiveIn(MachineBasicBlock & MBB)168 static bool isEAXLiveIn(MachineBasicBlock &MBB) {
169 for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) {
170 unsigned Reg = RegMask.PhysReg;
171
172 if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX ||
173 Reg == X86::AH || Reg == X86::AL)
174 return true;
175 }
176
177 return false;
178 }
179
180 /// Check if the flags need to be preserved before the terminators.
181 /// This would be the case, if the eflags is live-in of the region
182 /// composed by the terminators or live-out of that region, without
183 /// being defined by a terminator.
184 static bool
flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock & MBB)185 flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) {
186 for (const MachineInstr &MI : MBB.terminators()) {
187 bool BreakNext = false;
188 for (const MachineOperand &MO : MI.operands()) {
189 if (!MO.isReg())
190 continue;
191 Register Reg = MO.getReg();
192 if (Reg != X86::EFLAGS)
193 continue;
194
195 // This terminator needs an eflags that is not defined
196 // by a previous another terminator:
197 // EFLAGS is live-in of the region composed by the terminators.
198 if (!MO.isDef())
199 return true;
200 // This terminator defines the eflags, i.e., we don't need to preserve it.
201 // However, we still need to check this specific terminator does not
202 // read a live-in value.
203 BreakNext = true;
204 }
205 // We found a definition of the eflags, no need to preserve them.
206 if (BreakNext)
207 return false;
208 }
209
210 // None of the terminators use or define the eflags.
211 // Check if they are live-out, that would imply we need to preserve them.
212 for (const MachineBasicBlock *Succ : MBB.successors())
213 if (Succ->isLiveIn(X86::EFLAGS))
214 return true;
215
216 return false;
217 }
218
219 /// emitSPUpdate - Emit a series of instructions to increment / decrement the
220 /// stack pointer by a constant value.
emitSPUpdate(MachineBasicBlock & MBB,MachineBasicBlock::iterator & MBBI,const DebugLoc & DL,int64_t NumBytes,bool InEpilogue) const221 void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB,
222 MachineBasicBlock::iterator &MBBI,
223 const DebugLoc &DL,
224 int64_t NumBytes, bool InEpilogue) const {
225 bool isSub = NumBytes < 0;
226 uint64_t Offset = isSub ? -NumBytes : NumBytes;
227 MachineInstr::MIFlag Flag =
228 isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy;
229
230 uint64_t Chunk = (1LL << 31) - 1;
231
232 MachineFunction &MF = *MBB.getParent();
233 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
234 const X86TargetLowering &TLI = *STI.getTargetLowering();
235 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
236
237 // It's ok to not take into account large chunks when probing, as the
238 // allocation is split in smaller chunks anyway.
239 if (EmitInlineStackProbe && !InEpilogue) {
240
241 // This pseudo-instruction is going to be expanded, potentially using a
242 // loop, by inlineStackProbe().
243 BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING)).addImm(Offset);
244 return;
245 } else if (Offset > Chunk) {
246 // Rather than emit a long series of instructions for large offsets,
247 // load the offset into a register and do one sub/add
248 unsigned Reg = 0;
249 unsigned Rax = (unsigned)(Is64Bit ? X86::RAX : X86::EAX);
250
251 if (isSub && !isEAXLiveIn(MBB))
252 Reg = Rax;
253 else
254 Reg = TRI->findDeadCallerSavedReg(MBB, MBBI);
255
256 unsigned AddSubRROpc =
257 isSub ? getSUBrrOpcode(Is64Bit) : getADDrrOpcode(Is64Bit);
258 if (Reg) {
259 BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Offset)), Reg)
260 .addImm(Offset)
261 .setMIFlag(Flag);
262 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AddSubRROpc), StackPtr)
263 .addReg(StackPtr)
264 .addReg(Reg);
265 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
266 return;
267 } else if (Offset > 8 * Chunk) {
268 // If we would need more than 8 add or sub instructions (a >16GB stack
269 // frame), it's worth spilling RAX to materialize this immediate.
270 // pushq %rax
271 // movabsq +-$Offset+-SlotSize, %rax
272 // addq %rsp, %rax
273 // xchg %rax, (%rsp)
274 // movq (%rsp), %rsp
275 assert(Is64Bit && "can't have 32-bit 16GB stack frame");
276 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
277 .addReg(Rax, RegState::Kill)
278 .setMIFlag(Flag);
279 // Subtract is not commutative, so negate the offset and always use add.
280 // Subtract 8 less and add 8 more to account for the PUSH we just did.
281 if (isSub)
282 Offset = -(Offset - SlotSize);
283 else
284 Offset = Offset + SlotSize;
285 BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Offset)), Rax)
286 .addImm(Offset)
287 .setMIFlag(Flag);
288 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(X86::ADD64rr), Rax)
289 .addReg(Rax)
290 .addReg(StackPtr);
291 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
292 // Exchange the new SP in RAX with the top of the stack.
293 addRegOffset(
294 BuildMI(MBB, MBBI, DL, TII.get(X86::XCHG64rm), Rax).addReg(Rax),
295 StackPtr, false, 0);
296 // Load new SP from the top of the stack into RSP.
297 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), StackPtr),
298 StackPtr, false, 0);
299 return;
300 }
301 }
302
303 while (Offset) {
304 uint64_t ThisVal = std::min(Offset, Chunk);
305 if (ThisVal == SlotSize) {
306 // Use push / pop for slot sized adjustments as a size optimization. We
307 // need to find a dead register when using pop.
308 unsigned Reg = isSub
309 ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX)
310 : TRI->findDeadCallerSavedReg(MBB, MBBI);
311 if (Reg) {
312 unsigned Opc = isSub
313 ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r)
314 : (Is64Bit ? X86::POP64r : X86::POP32r);
315 BuildMI(MBB, MBBI, DL, TII.get(Opc))
316 .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub))
317 .setMIFlag(Flag);
318 Offset -= ThisVal;
319 continue;
320 }
321 }
322
323 BuildStackAdjustment(MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue)
324 .setMIFlag(Flag);
325
326 Offset -= ThisVal;
327 }
328 }
329
BuildStackAdjustment(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,int64_t Offset,bool InEpilogue) const330 MachineInstrBuilder X86FrameLowering::BuildStackAdjustment(
331 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
332 const DebugLoc &DL, int64_t Offset, bool InEpilogue) const {
333 assert(Offset != 0 && "zero offset stack adjustment requested");
334
335 // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue
336 // is tricky.
337 bool UseLEA;
338 if (!InEpilogue) {
339 // Check if inserting the prologue at the beginning
340 // of MBB would require to use LEA operations.
341 // We need to use LEA operations if EFLAGS is live in, because
342 // it means an instruction will read it before it gets defined.
343 UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS);
344 } else {
345 // If we can use LEA for SP but we shouldn't, check that none
346 // of the terminators uses the eflags. Otherwise we will insert
347 // a ADD that will redefine the eflags and break the condition.
348 // Alternatively, we could move the ADD, but this may not be possible
349 // and is an optimization anyway.
350 UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent());
351 if (UseLEA && !STI.useLeaForSP())
352 UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB);
353 // If that assert breaks, that means we do not do the right thing
354 // in canUseAsEpilogue.
355 assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) &&
356 "We shouldn't have allowed this insertion point");
357 }
358
359 MachineInstrBuilder MI;
360 if (UseLEA) {
361 MI = addRegOffset(BuildMI(MBB, MBBI, DL,
362 TII.get(getLEArOpcode(Uses64BitFramePtr)),
363 StackPtr),
364 StackPtr, false, Offset);
365 } else {
366 bool IsSub = Offset < 0;
367 uint64_t AbsOffset = IsSub ? -Offset : Offset;
368 const unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset)
369 : getADDriOpcode(Uses64BitFramePtr, AbsOffset);
370 MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
371 .addReg(StackPtr)
372 .addImm(AbsOffset);
373 MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead.
374 }
375 return MI;
376 }
377
mergeSPUpdates(MachineBasicBlock & MBB,MachineBasicBlock::iterator & MBBI,bool doMergeWithPrevious) const378 int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB,
379 MachineBasicBlock::iterator &MBBI,
380 bool doMergeWithPrevious) const {
381 if ((doMergeWithPrevious && MBBI == MBB.begin()) ||
382 (!doMergeWithPrevious && MBBI == MBB.end()))
383 return 0;
384
385 MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI;
386
387 PI = skipDebugInstructionsBackward(PI, MBB.begin());
388 // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI
389 // instruction, and that there are no DBG_VALUE or other instructions between
390 // ADD/SUB/LEA and its corresponding CFI instruction.
391 /* TODO: Add support for the case where there are multiple CFI instructions
392 below the ADD/SUB/LEA, e.g.:
393 ...
394 add
395 cfi_def_cfa_offset
396 cfi_offset
397 ...
398 */
399 if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction())
400 PI = std::prev(PI);
401
402 unsigned Opc = PI->getOpcode();
403 int Offset = 0;
404
405 if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 ||
406 Opc == X86::ADD32ri || Opc == X86::ADD32ri8) &&
407 PI->getOperand(0).getReg() == StackPtr){
408 assert(PI->getOperand(1).getReg() == StackPtr);
409 Offset = PI->getOperand(2).getImm();
410 } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) &&
411 PI->getOperand(0).getReg() == StackPtr &&
412 PI->getOperand(1).getReg() == StackPtr &&
413 PI->getOperand(2).getImm() == 1 &&
414 PI->getOperand(3).getReg() == X86::NoRegister &&
415 PI->getOperand(5).getReg() == X86::NoRegister) {
416 // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg.
417 Offset = PI->getOperand(4).getImm();
418 } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 ||
419 Opc == X86::SUB32ri || Opc == X86::SUB32ri8) &&
420 PI->getOperand(0).getReg() == StackPtr) {
421 assert(PI->getOperand(1).getReg() == StackPtr);
422 Offset = -PI->getOperand(2).getImm();
423 } else
424 return 0;
425
426 PI = MBB.erase(PI);
427 if (PI != MBB.end() && PI->isCFIInstruction()) {
428 auto CIs = MBB.getParent()->getFrameInstructions();
429 MCCFIInstruction CI = CIs[PI->getOperand(0).getCFIIndex()];
430 if (CI.getOperation() == MCCFIInstruction::OpDefCfaOffset ||
431 CI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset)
432 PI = MBB.erase(PI);
433 }
434 if (!doMergeWithPrevious)
435 MBBI = skipDebugInstructionsForward(PI, MBB.end());
436
437 return Offset;
438 }
439
BuildCFI(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,const MCCFIInstruction & CFIInst,MachineInstr::MIFlag Flag) const440 void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB,
441 MachineBasicBlock::iterator MBBI,
442 const DebugLoc &DL,
443 const MCCFIInstruction &CFIInst,
444 MachineInstr::MIFlag Flag) const {
445 MachineFunction &MF = *MBB.getParent();
446 unsigned CFIIndex = MF.addFrameInst(CFIInst);
447 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
448 .addCFIIndex(CFIIndex)
449 .setMIFlag(Flag);
450 }
451
452 /// Emits Dwarf Info specifying offsets of callee saved registers and
453 /// frame pointer. This is called only when basic block sections are enabled.
emitCalleeSavedFrameMovesFullCFA(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const454 void X86FrameLowering::emitCalleeSavedFrameMovesFullCFA(
455 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
456 MachineFunction &MF = *MBB.getParent();
457 if (!hasFP(MF)) {
458 emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true);
459 return;
460 }
461 const MachineModuleInfo &MMI = MF.getMMI();
462 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
463 const Register FramePtr = TRI->getFrameRegister(MF);
464 const Register MachineFramePtr =
465 STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(FramePtr, 64))
466 : FramePtr;
467 unsigned DwarfReg = MRI->getDwarfRegNum(MachineFramePtr, true);
468 // Offset = space for return address + size of the frame pointer itself.
469 unsigned Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4);
470 BuildCFI(MBB, MBBI, DebugLoc{},
471 MCCFIInstruction::createOffset(nullptr, DwarfReg, -Offset));
472 emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true);
473 }
474
emitCalleeSavedFrameMoves(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool IsPrologue) const475 void X86FrameLowering::emitCalleeSavedFrameMoves(
476 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
477 const DebugLoc &DL, bool IsPrologue) const {
478 MachineFunction &MF = *MBB.getParent();
479 MachineFrameInfo &MFI = MF.getFrameInfo();
480 MachineModuleInfo &MMI = MF.getMMI();
481 const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo();
482
483 // Add callee saved registers to move list.
484 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
485
486 // Calculate offsets.
487 for (const CalleeSavedInfo &I : CSI) {
488 int64_t Offset = MFI.getObjectOffset(I.getFrameIdx());
489 Register Reg = I.getReg();
490 unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
491
492 if (IsPrologue) {
493 BuildCFI(MBB, MBBI, DL,
494 MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset));
495 } else {
496 BuildCFI(MBB, MBBI, DL,
497 MCCFIInstruction::createRestore(nullptr, DwarfReg));
498 }
499 }
500 }
501
emitZeroCallUsedRegs(BitVector RegsToZero,MachineBasicBlock & MBB) const502 void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
503 MachineBasicBlock &MBB) const {
504 const MachineFunction &MF = *MBB.getParent();
505
506 // Insertion point.
507 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
508
509 // Fake a debug loc.
510 DebugLoc DL;
511 if (MBBI != MBB.end())
512 DL = MBBI->getDebugLoc();
513
514 // Zero out FP stack if referenced. Do this outside of the loop below so that
515 // it's done only once.
516 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
517 for (MCRegister Reg : RegsToZero.set_bits()) {
518 if (!X86::RFP80RegClass.contains(Reg))
519 continue;
520
521 unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
522 for (unsigned i = 0; i != NumFPRegs; ++i)
523 BuildMI(MBB, MBBI, DL, TII.get(X86::LD_F0));
524
525 for (unsigned i = 0; i != NumFPRegs; ++i)
526 BuildMI(MBB, MBBI, DL, TII.get(X86::ST_FPrr)).addReg(X86::ST0);
527 break;
528 }
529
530 // For GPRs, we only care to clear out the 32-bit register.
531 BitVector GPRsToZero(TRI->getNumRegs());
532 for (MCRegister Reg : RegsToZero.set_bits())
533 if (TRI->isGeneralPurposeRegister(MF, Reg)) {
534 GPRsToZero.set(getX86SubSuperRegisterOrZero(Reg, 32));
535 RegsToZero.reset(Reg);
536 }
537
538 for (MCRegister Reg : GPRsToZero.set_bits())
539 BuildMI(MBB, MBBI, DL, TII.get(X86::XOR32rr), Reg)
540 .addReg(Reg, RegState::Undef)
541 .addReg(Reg, RegState::Undef);
542
543 // Zero out registers.
544 for (MCRegister Reg : RegsToZero.set_bits()) {
545 if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
546 // FIXME: Ignore MMX registers?
547 continue;
548
549 unsigned XorOp;
550 if (X86::VR128RegClass.contains(Reg)) {
551 // XMM#
552 if (!ST.hasSSE1())
553 continue;
554 XorOp = X86::PXORrr;
555 } else if (X86::VR256RegClass.contains(Reg)) {
556 // YMM#
557 if (!ST.hasAVX())
558 continue;
559 XorOp = X86::VPXORrr;
560 } else if (X86::VR512RegClass.contains(Reg)) {
561 // ZMM#
562 if (!ST.hasAVX512())
563 continue;
564 XorOp = X86::VPXORYrr;
565 } else if (X86::VK1RegClass.contains(Reg) ||
566 X86::VK2RegClass.contains(Reg) ||
567 X86::VK4RegClass.contains(Reg) ||
568 X86::VK8RegClass.contains(Reg) ||
569 X86::VK16RegClass.contains(Reg)) {
570 if (!ST.hasVLX())
571 continue;
572 XorOp = ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
573 } else {
574 continue;
575 }
576
577 BuildMI(MBB, MBBI, DL, TII.get(XorOp), Reg)
578 .addReg(Reg, RegState::Undef)
579 .addReg(Reg, RegState::Undef);
580 }
581 }
582
emitStackProbe(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog,std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const583 void X86FrameLowering::emitStackProbe(
584 MachineFunction &MF, MachineBasicBlock &MBB,
585 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
586 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
587 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
588 if (STI.isTargetWindowsCoreCLR()) {
589 if (InProlog) {
590 BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING))
591 .addImm(0 /* no explicit stack size */);
592 } else {
593 emitStackProbeInline(MF, MBB, MBBI, DL, false);
594 }
595 } else {
596 emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum);
597 }
598 }
599
stackProbeFunctionModifiesSP() const600 bool X86FrameLowering::stackProbeFunctionModifiesSP() const {
601 return STI.isOSWindows() && !STI.isTargetWin64();
602 }
603
inlineStackProbe(MachineFunction & MF,MachineBasicBlock & PrologMBB) const604 void X86FrameLowering::inlineStackProbe(MachineFunction &MF,
605 MachineBasicBlock &PrologMBB) const {
606 auto Where = llvm::find_if(PrologMBB, [](MachineInstr &MI) {
607 return MI.getOpcode() == X86::STACKALLOC_W_PROBING;
608 });
609 if (Where != PrologMBB.end()) {
610 DebugLoc DL = PrologMBB.findDebugLoc(Where);
611 emitStackProbeInline(MF, PrologMBB, Where, DL, true);
612 Where->eraseFromParent();
613 }
614 }
615
emitStackProbeInline(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog) const616 void X86FrameLowering::emitStackProbeInline(MachineFunction &MF,
617 MachineBasicBlock &MBB,
618 MachineBasicBlock::iterator MBBI,
619 const DebugLoc &DL,
620 bool InProlog) const {
621 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
622 if (STI.isTargetWindowsCoreCLR() && STI.is64Bit())
623 emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog);
624 else
625 emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog);
626 }
627
emitStackProbeInlineGeneric(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog) const628 void X86FrameLowering::emitStackProbeInlineGeneric(
629 MachineFunction &MF, MachineBasicBlock &MBB,
630 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
631 MachineInstr &AllocWithProbe = *MBBI;
632 uint64_t Offset = AllocWithProbe.getOperand(0).getImm();
633
634 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
635 const X86TargetLowering &TLI = *STI.getTargetLowering();
636 assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) &&
637 "different expansion expected for CoreCLR 64 bit");
638
639 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
640 uint64_t ProbeChunk = StackProbeSize * 8;
641
642 uint64_t MaxAlign =
643 TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0;
644
645 // Synthesize a loop or unroll it, depending on the number of iterations.
646 // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left
647 // between the unaligned rsp and current rsp.
648 if (Offset > ProbeChunk) {
649 emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset,
650 MaxAlign % StackProbeSize);
651 } else {
652 emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset,
653 MaxAlign % StackProbeSize);
654 }
655 }
656
emitStackProbeInlineGenericBlock(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,uint64_t Offset,uint64_t AlignOffset) const657 void X86FrameLowering::emitStackProbeInlineGenericBlock(
658 MachineFunction &MF, MachineBasicBlock &MBB,
659 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
660 uint64_t AlignOffset) const {
661
662 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
663 const bool HasFP = hasFP(MF);
664 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
665 const X86TargetLowering &TLI = *STI.getTargetLowering();
666 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
667 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
668
669 uint64_t CurrentOffset = 0;
670
671 assert(AlignOffset < StackProbeSize);
672
673 // If the offset is so small it fits within a page, there's nothing to do.
674 if (StackProbeSize < Offset + AlignOffset) {
675
676 uint64_t StackAdjustment = StackProbeSize - AlignOffset;
677 BuildStackAdjustment(MBB, MBBI, DL, -StackAdjustment, /*InEpilogue=*/false)
678 .setMIFlag(MachineInstr::FrameSetup);
679 if (!HasFP && NeedsDwarfCFI) {
680 BuildCFI(
681 MBB, MBBI, DL,
682 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment));
683 }
684
685 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
686 .setMIFlag(MachineInstr::FrameSetup),
687 StackPtr, false, 0)
688 .addImm(0)
689 .setMIFlag(MachineInstr::FrameSetup);
690 NumFrameExtraProbe++;
691 CurrentOffset = StackProbeSize - AlignOffset;
692 }
693
694 // For the next N - 1 pages, just probe. I tried to take advantage of
695 // natural probes but it implies much more logic and there was very few
696 // interesting natural probes to interleave.
697 while (CurrentOffset + StackProbeSize < Offset) {
698 BuildStackAdjustment(MBB, MBBI, DL, -StackProbeSize, /*InEpilogue=*/false)
699 .setMIFlag(MachineInstr::FrameSetup);
700
701 if (!HasFP && NeedsDwarfCFI) {
702 BuildCFI(
703 MBB, MBBI, DL,
704 MCCFIInstruction::createAdjustCfaOffset(nullptr, StackProbeSize));
705 }
706 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
707 .setMIFlag(MachineInstr::FrameSetup),
708 StackPtr, false, 0)
709 .addImm(0)
710 .setMIFlag(MachineInstr::FrameSetup);
711 NumFrameExtraProbe++;
712 CurrentOffset += StackProbeSize;
713 }
714
715 // No need to probe the tail, it is smaller than a Page.
716 uint64_t ChunkSize = Offset - CurrentOffset;
717 if (ChunkSize == SlotSize) {
718 // Use push for slot sized adjustments as a size optimization,
719 // like emitSPUpdate does when not probing.
720 unsigned Reg = Is64Bit ? X86::RAX : X86::EAX;
721 unsigned Opc = Is64Bit ? X86::PUSH64r : X86::PUSH32r;
722 BuildMI(MBB, MBBI, DL, TII.get(Opc))
723 .addReg(Reg, RegState::Undef)
724 .setMIFlag(MachineInstr::FrameSetup);
725 } else {
726 BuildStackAdjustment(MBB, MBBI, DL, -ChunkSize, /*InEpilogue=*/false)
727 .setMIFlag(MachineInstr::FrameSetup);
728 }
729 // No need to adjust Dwarf CFA offset here, the last position of the stack has
730 // been defined
731 }
732
emitStackProbeInlineGenericLoop(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,uint64_t Offset,uint64_t AlignOffset) const733 void X86FrameLowering::emitStackProbeInlineGenericLoop(
734 MachineFunction &MF, MachineBasicBlock &MBB,
735 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset,
736 uint64_t AlignOffset) const {
737 assert(Offset && "null offset");
738
739 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
740 MachineBasicBlock::LQR_Live &&
741 "Inline stack probe loop will clobber live EFLAGS.");
742
743 const bool NeedsDwarfCFI = needsDwarfCFI(MF);
744 const bool HasFP = hasFP(MF);
745 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
746 const X86TargetLowering &TLI = *STI.getTargetLowering();
747 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
748 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
749
750 if (AlignOffset) {
751 if (AlignOffset < StackProbeSize) {
752 // Perform a first smaller allocation followed by a probe.
753 BuildStackAdjustment(MBB, MBBI, DL, -AlignOffset, /*InEpilogue=*/false)
754 .setMIFlag(MachineInstr::FrameSetup);
755
756 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc))
757 .setMIFlag(MachineInstr::FrameSetup),
758 StackPtr, false, 0)
759 .addImm(0)
760 .setMIFlag(MachineInstr::FrameSetup);
761 NumFrameExtraProbe++;
762 Offset -= AlignOffset;
763 }
764 }
765
766 // Synthesize a loop
767 NumFrameLoopProbe++;
768 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
769
770 MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(LLVM_BB);
771 MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(LLVM_BB);
772
773 MachineFunction::iterator MBBIter = ++MBB.getIterator();
774 MF.insert(MBBIter, testMBB);
775 MF.insert(MBBIter, tailMBB);
776
777 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
778 : Is64Bit ? X86::R11D
779 : X86::EAX;
780
781 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
782 .addReg(StackPtr)
783 .setMIFlag(MachineInstr::FrameSetup);
784
785 // save loop bound
786 {
787 const unsigned BoundOffset = alignDown(Offset, StackProbeSize);
788 const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr, BoundOffset);
789 BuildMI(MBB, MBBI, DL, TII.get(SUBOpc), FinalStackProbed)
790 .addReg(FinalStackProbed)
791 .addImm(BoundOffset)
792 .setMIFlag(MachineInstr::FrameSetup);
793
794 // while in the loop, use loop-invariant reg for CFI,
795 // instead of the stack pointer, which changes during the loop
796 if (!HasFP && NeedsDwarfCFI) {
797 // x32 uses the same DWARF register numbers as x86-64,
798 // so there isn't a register number for r11d, we must use r11 instead
799 const Register DwarfFinalStackProbed =
800 STI.isTarget64BitILP32()
801 ? Register(getX86SubSuperRegister(FinalStackProbed, 64))
802 : FinalStackProbed;
803
804 BuildCFI(MBB, MBBI, DL,
805 MCCFIInstruction::createDefCfaRegister(
806 nullptr, TRI->getDwarfRegNum(DwarfFinalStackProbed, true)));
807 BuildCFI(MBB, MBBI, DL,
808 MCCFIInstruction::createAdjustCfaOffset(nullptr, BoundOffset));
809 }
810 }
811
812 // allocate a page
813 BuildStackAdjustment(*testMBB, testMBB->end(), DL, -StackProbeSize,
814 /*InEpilogue=*/false)
815 .setMIFlag(MachineInstr::FrameSetup);
816
817 // touch the page
818 addRegOffset(BuildMI(testMBB, DL, TII.get(MovMIOpc))
819 .setMIFlag(MachineInstr::FrameSetup),
820 StackPtr, false, 0)
821 .addImm(0)
822 .setMIFlag(MachineInstr::FrameSetup);
823
824 // cmp with stack pointer bound
825 BuildMI(testMBB, DL, TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
826 .addReg(StackPtr)
827 .addReg(FinalStackProbed)
828 .setMIFlag(MachineInstr::FrameSetup);
829
830 // jump
831 BuildMI(testMBB, DL, TII.get(X86::JCC_1))
832 .addMBB(testMBB)
833 .addImm(X86::COND_NE)
834 .setMIFlag(MachineInstr::FrameSetup);
835 testMBB->addSuccessor(testMBB);
836 testMBB->addSuccessor(tailMBB);
837
838 // BB management
839 tailMBB->splice(tailMBB->end(), &MBB, MBBI, MBB.end());
840 tailMBB->transferSuccessorsAndUpdatePHIs(&MBB);
841 MBB.addSuccessor(testMBB);
842
843 // handle tail
844 const uint64_t TailOffset = Offset % StackProbeSize;
845 MachineBasicBlock::iterator TailMBBIter = tailMBB->begin();
846 if (TailOffset) {
847 BuildStackAdjustment(*tailMBB, TailMBBIter, DL, -TailOffset,
848 /*InEpilogue=*/false)
849 .setMIFlag(MachineInstr::FrameSetup);
850 }
851
852 // after the loop, switch back to stack pointer for CFI
853 if (!HasFP && NeedsDwarfCFI) {
854 // x32 uses the same DWARF register numbers as x86-64,
855 // so there isn't a register number for esp, we must use rsp instead
856 const Register DwarfStackPtr =
857 STI.isTarget64BitILP32()
858 ? Register(getX86SubSuperRegister(StackPtr, 64))
859 : Register(StackPtr);
860
861 BuildCFI(*tailMBB, TailMBBIter, DL,
862 MCCFIInstruction::createDefCfaRegister(
863 nullptr, TRI->getDwarfRegNum(DwarfStackPtr, true)));
864 }
865
866 // Update Live In information
867 recomputeLiveIns(*testMBB);
868 recomputeLiveIns(*tailMBB);
869 }
870
emitStackProbeInlineWindowsCoreCLR64(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog) const871 void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
872 MachineFunction &MF, MachineBasicBlock &MBB,
873 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const {
874 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
875 assert(STI.is64Bit() && "different expansion needed for 32 bit");
876 assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR");
877 const TargetInstrInfo &TII = *STI.getInstrInfo();
878 const BasicBlock *LLVM_BB = MBB.getBasicBlock();
879
880 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
881 MachineBasicBlock::LQR_Live &&
882 "Inline stack probe loop will clobber live EFLAGS.");
883
884 // RAX contains the number of bytes of desired stack adjustment.
885 // The handling here assumes this value has already been updated so as to
886 // maintain stack alignment.
887 //
888 // We need to exit with RSP modified by this amount and execute suitable
889 // page touches to notify the OS that we're growing the stack responsibly.
890 // All stack probing must be done without modifying RSP.
891 //
892 // MBB:
893 // SizeReg = RAX;
894 // ZeroReg = 0
895 // CopyReg = RSP
896 // Flags, TestReg = CopyReg - SizeReg
897 // FinalReg = !Flags.Ovf ? TestReg : ZeroReg
898 // LimitReg = gs magic thread env access
899 // if FinalReg >= LimitReg goto ContinueMBB
900 // RoundBB:
901 // RoundReg = page address of FinalReg
902 // LoopMBB:
903 // LoopReg = PHI(LimitReg,ProbeReg)
904 // ProbeReg = LoopReg - PageSize
905 // [ProbeReg] = 0
906 // if (ProbeReg > RoundReg) goto LoopMBB
907 // ContinueMBB:
908 // RSP = RSP - RAX
909 // [rest of original MBB]
910
911 // Set up the new basic blocks
912 MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB);
913 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
914 MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB);
915
916 MachineFunction::iterator MBBIter = std::next(MBB.getIterator());
917 MF.insert(MBBIter, RoundMBB);
918 MF.insert(MBBIter, LoopMBB);
919 MF.insert(MBBIter, ContinueMBB);
920
921 // Split MBB and move the tail portion down to ContinueMBB.
922 MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI);
923 ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end());
924 ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB);
925
926 // Some useful constants
927 const int64_t ThreadEnvironmentStackLimit = 0x10;
928 const int64_t PageSize = 0x1000;
929 const int64_t PageMask = ~(PageSize - 1);
930
931 // Registers we need. For the normal case we use virtual
932 // registers. For the prolog expansion we use RAX, RCX and RDX.
933 MachineRegisterInfo &MRI = MF.getRegInfo();
934 const TargetRegisterClass *RegClass = &X86::GR64RegClass;
935 const Register SizeReg = InProlog ? X86::RAX
936 : MRI.createVirtualRegister(RegClass),
937 ZeroReg = InProlog ? X86::RCX
938 : MRI.createVirtualRegister(RegClass),
939 CopyReg = InProlog ? X86::RDX
940 : MRI.createVirtualRegister(RegClass),
941 TestReg = InProlog ? X86::RDX
942 : MRI.createVirtualRegister(RegClass),
943 FinalReg = InProlog ? X86::RDX
944 : MRI.createVirtualRegister(RegClass),
945 RoundedReg = InProlog ? X86::RDX
946 : MRI.createVirtualRegister(RegClass),
947 LimitReg = InProlog ? X86::RCX
948 : MRI.createVirtualRegister(RegClass),
949 JoinReg = InProlog ? X86::RCX
950 : MRI.createVirtualRegister(RegClass),
951 ProbeReg = InProlog ? X86::RCX
952 : MRI.createVirtualRegister(RegClass);
953
954 // SP-relative offsets where we can save RCX and RDX.
955 int64_t RCXShadowSlot = 0;
956 int64_t RDXShadowSlot = 0;
957
958 // If inlining in the prolog, save RCX and RDX.
959 if (InProlog) {
960 // Compute the offsets. We need to account for things already
961 // pushed onto the stack at this point: return address, frame
962 // pointer (if used), and callee saves.
963 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
964 const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize();
965 const bool HasFP = hasFP(MF);
966
967 // Check if we need to spill RCX and/or RDX.
968 // Here we assume that no earlier prologue instruction changes RCX and/or
969 // RDX, so checking the block live-ins is enough.
970 const bool IsRCXLiveIn = MBB.isLiveIn(X86::RCX);
971 const bool IsRDXLiveIn = MBB.isLiveIn(X86::RDX);
972 int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0);
973 // Assign the initial slot to both registers, then change RDX's slot if both
974 // need to be spilled.
975 if (IsRCXLiveIn)
976 RCXShadowSlot = InitSlot;
977 if (IsRDXLiveIn)
978 RDXShadowSlot = InitSlot;
979 if (IsRDXLiveIn && IsRCXLiveIn)
980 RDXShadowSlot += 8;
981 // Emit the saves if needed.
982 if (IsRCXLiveIn)
983 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
984 RCXShadowSlot)
985 .addReg(X86::RCX);
986 if (IsRDXLiveIn)
987 addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false,
988 RDXShadowSlot)
989 .addReg(X86::RDX);
990 } else {
991 // Not in the prolog. Copy RAX to a virtual reg.
992 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX);
993 }
994
995 // Add code to MBB to check for overflow and set the new target stack pointer
996 // to zero if so.
997 BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg)
998 .addReg(ZeroReg, RegState::Undef)
999 .addReg(ZeroReg, RegState::Undef);
1000 BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP);
1001 BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg)
1002 .addReg(CopyReg)
1003 .addReg(SizeReg);
1004 BuildMI(&MBB, DL, TII.get(X86::CMOV64rr), FinalReg)
1005 .addReg(TestReg)
1006 .addReg(ZeroReg)
1007 .addImm(X86::COND_B);
1008
1009 // FinalReg now holds final stack pointer value, or zero if
1010 // allocation would overflow. Compare against the current stack
1011 // limit from the thread environment block. Note this limit is the
1012 // lowest touched page on the stack, not the point at which the OS
1013 // will cause an overflow exception, so this is just an optimization
1014 // to avoid unnecessarily touching pages that are below the current
1015 // SP but already committed to the stack by the OS.
1016 BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg)
1017 .addReg(0)
1018 .addImm(1)
1019 .addReg(0)
1020 .addImm(ThreadEnvironmentStackLimit)
1021 .addReg(X86::GS);
1022 BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg);
1023 // Jump if the desired stack pointer is at or above the stack limit.
1024 BuildMI(&MBB, DL, TII.get(X86::JCC_1)).addMBB(ContinueMBB).addImm(X86::COND_AE);
1025
1026 // Add code to roundMBB to round the final stack pointer to a page boundary.
1027 RoundMBB->addLiveIn(FinalReg);
1028 BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg)
1029 .addReg(FinalReg)
1030 .addImm(PageMask);
1031 BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB);
1032
1033 // LimitReg now holds the current stack limit, RoundedReg page-rounded
1034 // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page
1035 // and probe until we reach RoundedReg.
1036 if (!InProlog) {
1037 BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg)
1038 .addReg(LimitReg)
1039 .addMBB(RoundMBB)
1040 .addReg(ProbeReg)
1041 .addMBB(LoopMBB);
1042 }
1043
1044 LoopMBB->addLiveIn(JoinReg);
1045 addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg,
1046 false, -PageSize);
1047
1048 // Probe by storing a byte onto the stack.
1049 BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi))
1050 .addReg(ProbeReg)
1051 .addImm(1)
1052 .addReg(0)
1053 .addImm(0)
1054 .addReg(0)
1055 .addImm(0);
1056
1057 LoopMBB->addLiveIn(RoundedReg);
1058 BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr))
1059 .addReg(RoundedReg)
1060 .addReg(ProbeReg);
1061 BuildMI(LoopMBB, DL, TII.get(X86::JCC_1)).addMBB(LoopMBB).addImm(X86::COND_NE);
1062
1063 MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI();
1064
1065 // If in prolog, restore RDX and RCX.
1066 if (InProlog) {
1067 if (RCXShadowSlot) // It means we spilled RCX in the prologue.
1068 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL,
1069 TII.get(X86::MOV64rm), X86::RCX),
1070 X86::RSP, false, RCXShadowSlot);
1071 if (RDXShadowSlot) // It means we spilled RDX in the prologue.
1072 addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL,
1073 TII.get(X86::MOV64rm), X86::RDX),
1074 X86::RSP, false, RDXShadowSlot);
1075 }
1076
1077 // Now that the probing is done, add code to continueMBB to update
1078 // the stack pointer for real.
1079 ContinueMBB->addLiveIn(SizeReg);
1080 BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP)
1081 .addReg(X86::RSP)
1082 .addReg(SizeReg);
1083
1084 // Add the control flow edges we need.
1085 MBB.addSuccessor(ContinueMBB);
1086 MBB.addSuccessor(RoundMBB);
1087 RoundMBB->addSuccessor(LoopMBB);
1088 LoopMBB->addSuccessor(ContinueMBB);
1089 LoopMBB->addSuccessor(LoopMBB);
1090
1091 // Mark all the instructions added to the prolog as frame setup.
1092 if (InProlog) {
1093 for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) {
1094 BeforeMBBI->setFlag(MachineInstr::FrameSetup);
1095 }
1096 for (MachineInstr &MI : *RoundMBB) {
1097 MI.setFlag(MachineInstr::FrameSetup);
1098 }
1099 for (MachineInstr &MI : *LoopMBB) {
1100 MI.setFlag(MachineInstr::FrameSetup);
1101 }
1102 for (MachineInstr &MI :
1103 llvm::make_range(ContinueMBB->begin(), ContinueMBBI)) {
1104 MI.setFlag(MachineInstr::FrameSetup);
1105 }
1106 }
1107 }
1108
emitStackProbeCall(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool InProlog,std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const1109 void X86FrameLowering::emitStackProbeCall(
1110 MachineFunction &MF, MachineBasicBlock &MBB,
1111 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog,
1112 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum) const {
1113 bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large;
1114
1115 // FIXME: Add indirect thunk support and remove this.
1116 if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls())
1117 report_fatal_error("Emitting stack probe calls on 64-bit with the large "
1118 "code model and indirect thunks not yet implemented.");
1119
1120 assert(MBB.computeRegisterLiveness(TRI, X86::EFLAGS, MBBI) !=
1121 MachineBasicBlock::LQR_Live &&
1122 "Stack probe calls will clobber live EFLAGS.");
1123
1124 unsigned CallOp;
1125 if (Is64Bit)
1126 CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32;
1127 else
1128 CallOp = X86::CALLpcrel32;
1129
1130 StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF);
1131
1132 MachineInstrBuilder CI;
1133 MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI);
1134
1135 // All current stack probes take AX and SP as input, clobber flags, and
1136 // preserve all registers. x86_64 probes leave RSP unmodified.
1137 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
1138 // For the large code model, we have to call through a register. Use R11,
1139 // as it is scratch in all supported calling conventions.
1140 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11)
1141 .addExternalSymbol(MF.createExternalSymbolName(Symbol));
1142 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11);
1143 } else {
1144 CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp))
1145 .addExternalSymbol(MF.createExternalSymbolName(Symbol));
1146 }
1147
1148 unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX;
1149 unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP;
1150 CI.addReg(AX, RegState::Implicit)
1151 .addReg(SP, RegState::Implicit)
1152 .addReg(AX, RegState::Define | RegState::Implicit)
1153 .addReg(SP, RegState::Define | RegState::Implicit)
1154 .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit);
1155
1156 MachineInstr *ModInst = CI;
1157 if (STI.isTargetWin64() || !STI.isOSWindows()) {
1158 // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves.
1159 // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp
1160 // themselves. They also does not clobber %rax so we can reuse it when
1161 // adjusting %rsp.
1162 // All other platforms do not specify a particular ABI for the stack probe
1163 // function, so we arbitrarily define it to not adjust %esp/%rsp itself.
1164 ModInst =
1165 BuildMI(MBB, MBBI, DL, TII.get(getSUBrrOpcode(Uses64BitFramePtr)), SP)
1166 .addReg(SP)
1167 .addReg(AX);
1168 }
1169
1170 // DebugInfo variable locations -- if there's an instruction number for the
1171 // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that
1172 // modifies SP.
1173 if (InstrNum) {
1174 if (STI.isTargetWin64() || !STI.isOSWindows()) {
1175 // Label destination operand of the subtract.
1176 MF.makeDebugValueSubstitution(*InstrNum,
1177 {ModInst->getDebugInstrNum(), 0});
1178 } else {
1179 // Label the call. The operand number is the penultimate operand, zero
1180 // based.
1181 unsigned SPDefOperand = ModInst->getNumOperands() - 2;
1182 MF.makeDebugValueSubstitution(
1183 *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand});
1184 }
1185 }
1186
1187 if (InProlog) {
1188 // Apply the frame setup flag to all inserted instrs.
1189 for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI)
1190 ExpansionMBBI->setFlag(MachineInstr::FrameSetup);
1191 }
1192 }
1193
calculateSetFPREG(uint64_t SPAdjust)1194 static unsigned calculateSetFPREG(uint64_t SPAdjust) {
1195 // Win64 ABI has a less restrictive limitation of 240; 128 works equally well
1196 // and might require smaller successive adjustments.
1197 const uint64_t Win64MaxSEHOffset = 128;
1198 uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset);
1199 // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode.
1200 return SEHFrameOffset & -16;
1201 }
1202
1203 // If we're forcing a stack realignment we can't rely on just the frame
1204 // info, we need to know the ABI stack alignment as well in case we
1205 // have a call out. Otherwise just make sure we have some alignment - we'll
1206 // go with the minimum SlotSize.
calculateMaxStackAlign(const MachineFunction & MF) const1207 uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const {
1208 const MachineFrameInfo &MFI = MF.getFrameInfo();
1209 Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment.
1210 Align StackAlign = getStackAlign();
1211 if (MF.getFunction().hasFnAttribute("stackrealign")) {
1212 if (MFI.hasCalls())
1213 MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
1214 else if (MaxAlign < SlotSize)
1215 MaxAlign = Align(SlotSize);
1216 }
1217 return MaxAlign.value();
1218 }
1219
BuildStackAlignAND(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,unsigned Reg,uint64_t MaxAlign) const1220 void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
1221 MachineBasicBlock::iterator MBBI,
1222 const DebugLoc &DL, unsigned Reg,
1223 uint64_t MaxAlign) const {
1224 uint64_t Val = -MaxAlign;
1225 unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val);
1226
1227 MachineFunction &MF = *MBB.getParent();
1228 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
1229 const X86TargetLowering &TLI = *STI.getTargetLowering();
1230 const uint64_t StackProbeSize = TLI.getStackProbeSize(MF);
1231 const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF);
1232
1233 // We want to make sure that (in worst case) less than StackProbeSize bytes
1234 // are not probed after the AND. This assumption is used in
1235 // emitStackProbeInlineGeneric.
1236 if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) {
1237 {
1238 NumFrameLoopProbe++;
1239 MachineBasicBlock *entryMBB =
1240 MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1241 MachineBasicBlock *headMBB =
1242 MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1243 MachineBasicBlock *bodyMBB =
1244 MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1245 MachineBasicBlock *footMBB =
1246 MF.CreateMachineBasicBlock(MBB.getBasicBlock());
1247
1248 MachineFunction::iterator MBBIter = MBB.getIterator();
1249 MF.insert(MBBIter, entryMBB);
1250 MF.insert(MBBIter, headMBB);
1251 MF.insert(MBBIter, bodyMBB);
1252 MF.insert(MBBIter, footMBB);
1253 const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi;
1254 Register FinalStackProbed = Uses64BitFramePtr ? X86::R11
1255 : Is64Bit ? X86::R11D
1256 : X86::EAX;
1257
1258 // Setup entry block
1259 {
1260
1261 entryMBB->splice(entryMBB->end(), &MBB, MBB.begin(), MBBI);
1262 BuildMI(entryMBB, DL, TII.get(TargetOpcode::COPY), FinalStackProbed)
1263 .addReg(StackPtr)
1264 .setMIFlag(MachineInstr::FrameSetup);
1265 MachineInstr *MI =
1266 BuildMI(entryMBB, DL, TII.get(AndOp), FinalStackProbed)
1267 .addReg(FinalStackProbed)
1268 .addImm(Val)
1269 .setMIFlag(MachineInstr::FrameSetup);
1270
1271 // The EFLAGS implicit def is dead.
1272 MI->getOperand(3).setIsDead();
1273
1274 BuildMI(entryMBB, DL,
1275 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1276 .addReg(FinalStackProbed)
1277 .addReg(StackPtr)
1278 .setMIFlag(MachineInstr::FrameSetup);
1279 BuildMI(entryMBB, DL, TII.get(X86::JCC_1))
1280 .addMBB(&MBB)
1281 .addImm(X86::COND_E)
1282 .setMIFlag(MachineInstr::FrameSetup);
1283 entryMBB->addSuccessor(headMBB);
1284 entryMBB->addSuccessor(&MBB);
1285 }
1286
1287 // Loop entry block
1288
1289 {
1290 const unsigned SUBOpc =
1291 getSUBriOpcode(Uses64BitFramePtr, StackProbeSize);
1292 BuildMI(headMBB, DL, TII.get(SUBOpc), StackPtr)
1293 .addReg(StackPtr)
1294 .addImm(StackProbeSize)
1295 .setMIFlag(MachineInstr::FrameSetup);
1296
1297 BuildMI(headMBB, DL,
1298 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1299 .addReg(StackPtr)
1300 .addReg(FinalStackProbed)
1301 .setMIFlag(MachineInstr::FrameSetup);
1302
1303 // jump to the footer if StackPtr < FinalStackProbed
1304 BuildMI(headMBB, DL, TII.get(X86::JCC_1))
1305 .addMBB(footMBB)
1306 .addImm(X86::COND_B)
1307 .setMIFlag(MachineInstr::FrameSetup);
1308
1309 headMBB->addSuccessor(bodyMBB);
1310 headMBB->addSuccessor(footMBB);
1311 }
1312
1313 // setup loop body
1314 {
1315 addRegOffset(BuildMI(bodyMBB, DL, TII.get(MovMIOpc))
1316 .setMIFlag(MachineInstr::FrameSetup),
1317 StackPtr, false, 0)
1318 .addImm(0)
1319 .setMIFlag(MachineInstr::FrameSetup);
1320
1321 const unsigned SUBOpc =
1322 getSUBriOpcode(Uses64BitFramePtr, StackProbeSize);
1323 BuildMI(bodyMBB, DL, TII.get(SUBOpc), StackPtr)
1324 .addReg(StackPtr)
1325 .addImm(StackProbeSize)
1326 .setMIFlag(MachineInstr::FrameSetup);
1327
1328 // cmp with stack pointer bound
1329 BuildMI(bodyMBB, DL,
1330 TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
1331 .addReg(FinalStackProbed)
1332 .addReg(StackPtr)
1333 .setMIFlag(MachineInstr::FrameSetup);
1334
1335 // jump back while FinalStackProbed < StackPtr
1336 BuildMI(bodyMBB, DL, TII.get(X86::JCC_1))
1337 .addMBB(bodyMBB)
1338 .addImm(X86::COND_B)
1339 .setMIFlag(MachineInstr::FrameSetup);
1340 bodyMBB->addSuccessor(bodyMBB);
1341 bodyMBB->addSuccessor(footMBB);
1342 }
1343
1344 // setup loop footer
1345 {
1346 BuildMI(footMBB, DL, TII.get(TargetOpcode::COPY), StackPtr)
1347 .addReg(FinalStackProbed)
1348 .setMIFlag(MachineInstr::FrameSetup);
1349 addRegOffset(BuildMI(footMBB, DL, TII.get(MovMIOpc))
1350 .setMIFlag(MachineInstr::FrameSetup),
1351 StackPtr, false, 0)
1352 .addImm(0)
1353 .setMIFlag(MachineInstr::FrameSetup);
1354 footMBB->addSuccessor(&MBB);
1355 }
1356
1357 recomputeLiveIns(*headMBB);
1358 recomputeLiveIns(*bodyMBB);
1359 recomputeLiveIns(*footMBB);
1360 recomputeLiveIns(MBB);
1361 }
1362 } else {
1363 MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg)
1364 .addReg(Reg)
1365 .addImm(Val)
1366 .setMIFlag(MachineInstr::FrameSetup);
1367
1368 // The EFLAGS implicit def is dead.
1369 MI->getOperand(3).setIsDead();
1370 }
1371 }
1372
1373 // FIXME: Get this from tablegen.
get64BitArgumentGPRs(CallingConv::ID CallConv,const X86Subtarget & Subtarget)1374 static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
1375 const X86Subtarget &Subtarget) {
1376 assert(Subtarget.is64Bit());
1377
1378 if (Subtarget.isCallingConvWin64(CallConv)) {
1379 static const MCPhysReg GPR64ArgRegsWin64[] = {
1380 X86::RCX, X86::RDX, X86::R8, X86::R9
1381 };
1382 return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
1383 }
1384
1385 static const MCPhysReg GPR64ArgRegs64Bit[] = {
1386 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1387 };
1388 return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
1389 }
1390
has128ByteRedZone(const MachineFunction & MF) const1391 bool X86FrameLowering::has128ByteRedZone(const MachineFunction& MF) const {
1392 // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be
1393 // clobbered by any interrupt handler.
1394 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1395 "MF used frame lowering for wrong subtarget");
1396 const Function &Fn = MF.getFunction();
1397 const bool IsWin64CC = STI.isCallingConvWin64(Fn.getCallingConv());
1398 return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Attribute::NoRedZone);
1399 }
1400
1401 /// Return true if we need to use the restricted Windows x64 prologue and
1402 /// epilogue code patterns that can be described with WinCFI (.seh_*
1403 /// directives).
isWin64Prologue(const MachineFunction & MF) const1404 bool X86FrameLowering::isWin64Prologue(const MachineFunction &MF) const {
1405 return MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
1406 }
1407
needsDwarfCFI(const MachineFunction & MF) const1408 bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const {
1409 return !isWin64Prologue(MF) && MF.needsFrameMoves();
1410 }
1411
1412 /// emitPrologue - Push callee-saved registers onto the stack, which
1413 /// automatically adjust the stack pointer. Adjust the stack pointer to allocate
1414 /// space for local variables. Also emit labels used by the exception handler to
1415 /// generate the exception handling frames.
1416
1417 /*
1418 Here's a gist of what gets emitted:
1419
1420 ; Establish frame pointer, if needed
1421 [if needs FP]
1422 push %rbp
1423 .cfi_def_cfa_offset 16
1424 .cfi_offset %rbp, -16
1425 .seh_pushreg %rpb
1426 mov %rsp, %rbp
1427 .cfi_def_cfa_register %rbp
1428
1429 ; Spill general-purpose registers
1430 [for all callee-saved GPRs]
1431 pushq %<reg>
1432 [if not needs FP]
1433 .cfi_def_cfa_offset (offset from RETADDR)
1434 .seh_pushreg %<reg>
1435
1436 ; If the required stack alignment > default stack alignment
1437 ; rsp needs to be re-aligned. This creates a "re-alignment gap"
1438 ; of unknown size in the stack frame.
1439 [if stack needs re-alignment]
1440 and $MASK, %rsp
1441
1442 ; Allocate space for locals
1443 [if target is Windows and allocated space > 4096 bytes]
1444 ; Windows needs special care for allocations larger
1445 ; than one page.
1446 mov $NNN, %rax
1447 call ___chkstk_ms/___chkstk
1448 sub %rax, %rsp
1449 [else]
1450 sub $NNN, %rsp
1451
1452 [if needs FP]
1453 .seh_stackalloc (size of XMM spill slots)
1454 .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots
1455 [else]
1456 .seh_stackalloc NNN
1457
1458 ; Spill XMMs
1459 ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved,
1460 ; they may get spilled on any platform, if the current function
1461 ; calls @llvm.eh.unwind.init
1462 [if needs FP]
1463 [for all callee-saved XMM registers]
1464 movaps %<xmm reg>, -MMM(%rbp)
1465 [for all callee-saved XMM registers]
1466 .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset)
1467 ; i.e. the offset relative to (%rbp - SEHFrameOffset)
1468 [else]
1469 [for all callee-saved XMM registers]
1470 movaps %<xmm reg>, KKK(%rsp)
1471 [for all callee-saved XMM registers]
1472 .seh_savexmm %<xmm reg>, KKK
1473
1474 .seh_endprologue
1475
1476 [if needs base pointer]
1477 mov %rsp, %rbx
1478 [if needs to restore base pointer]
1479 mov %rsp, -MMM(%rbp)
1480
1481 ; Emit CFI info
1482 [if needs FP]
1483 [for all callee-saved registers]
1484 .cfi_offset %<reg>, (offset from %rbp)
1485 [else]
1486 .cfi_def_cfa_offset (offset from RETADDR)
1487 [for all callee-saved registers]
1488 .cfi_offset %<reg>, (offset from %rsp)
1489
1490 Notes:
1491 - .seh directives are emitted only for Windows 64 ABI
1492 - .cv_fpo directives are emitted on win32 when emitting CodeView
1493 - .cfi directives are emitted for all other ABIs
1494 - for 32-bit code, substitute %e?? registers for %r??
1495 */
1496
emitPrologue(MachineFunction & MF,MachineBasicBlock & MBB) const1497 void X86FrameLowering::emitPrologue(MachineFunction &MF,
1498 MachineBasicBlock &MBB) const {
1499 assert(&STI == &MF.getSubtarget<X86Subtarget>() &&
1500 "MF used frame lowering for wrong subtarget");
1501 MachineBasicBlock::iterator MBBI = MBB.begin();
1502 MachineFrameInfo &MFI = MF.getFrameInfo();
1503 const Function &Fn = MF.getFunction();
1504 MachineModuleInfo &MMI = MF.getMMI();
1505 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
1506 uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment.
1507 uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate.
1508 bool IsFunclet = MBB.isEHFuncletEntry();
1509 EHPersonality Personality = EHPersonality::Unknown;
1510 if (Fn.hasPersonalityFn())
1511 Personality = classifyEHPersonality(Fn.getPersonalityFn());
1512 bool FnHasClrFunclet =
1513 MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR;
1514 bool IsClrFunclet = IsFunclet && FnHasClrFunclet;
1515 bool HasFP = hasFP(MF);
1516 bool IsWin64Prologue = isWin64Prologue(MF);
1517 bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry();
1518 // FIXME: Emit FPO data for EH funclets.
1519 bool NeedsWinFPO =
1520 !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag();
1521 bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO;
1522 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1523 Register FramePtr = TRI->getFrameRegister(MF);
1524 const Register MachineFramePtr =
1525 STI.isTarget64BitILP32()
1526 ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr;
1527 Register BasePtr = TRI->getBaseRegister();
1528 bool HasWinCFI = false;
1529
1530 // Debug location must be unknown since the first debug location is used
1531 // to determine the end of the prologue.
1532 DebugLoc DL;
1533
1534 // Space reserved for stack-based arguments when making a (ABI-guaranteed)
1535 // tail call.
1536 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
1537 if (TailCallArgReserveSize && IsWin64Prologue)
1538 report_fatal_error("Can't handle guaranteed tail call under win64 yet");
1539
1540 const bool EmitStackProbeCall =
1541 STI.getTargetLowering()->hasStackProbeSymbol(MF);
1542 unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF);
1543
1544 if (HasFP && X86FI->hasSwiftAsyncContext()) {
1545 switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
1546 case SwiftAsyncFramePointerMode::DeploymentBased:
1547 if (STI.swiftAsyncContextIsDynamicallySet()) {
1548 // The special symbol below is absolute and has a *value* suitable to be
1549 // combined with the frame pointer directly.
1550 BuildMI(MBB, MBBI, DL, TII.get(X86::OR64rm), MachineFramePtr)
1551 .addUse(MachineFramePtr)
1552 .addUse(X86::RIP)
1553 .addImm(1)
1554 .addUse(X86::NoRegister)
1555 .addExternalSymbol("swift_async_extendedFramePointerFlags",
1556 X86II::MO_GOTPCREL)
1557 .addUse(X86::NoRegister);
1558 break;
1559 }
1560 [[fallthrough]];
1561
1562 case SwiftAsyncFramePointerMode::Always:
1563 BuildMI(MBB, MBBI, DL, TII.get(X86::BTS64ri8), MachineFramePtr)
1564 .addUse(MachineFramePtr)
1565 .addImm(60)
1566 .setMIFlag(MachineInstr::FrameSetup);
1567 break;
1568
1569 case SwiftAsyncFramePointerMode::Never:
1570 break;
1571 }
1572 }
1573
1574 // Re-align the stack on 64-bit if the x86-interrupt calling convention is
1575 // used and an error code was pushed, since the x86-64 ABI requires a 16-byte
1576 // stack alignment.
1577 if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit &&
1578 Fn.arg_size() == 2) {
1579 StackSize += 8;
1580 MFI.setStackSize(StackSize);
1581
1582 // Update the stack pointer by pushing a register. This is the instruction
1583 // emitted that would be end up being emitted by a call to `emitSPUpdate`.
1584 // Hard-coding the update to a push avoids emitting a second
1585 // `STACKALLOC_W_PROBING` instruction in the save block: We know that stack
1586 // probing isn't needed anyways for an 8-byte update.
1587 // Pushing a register leaves us in a similar situation to a regular
1588 // function call where we know that the address at (rsp-8) is writeable.
1589 // That way we avoid any off-by-ones with stack probing for additional
1590 // stack pointer updates later on.
1591 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1592 .addReg(X86::RAX, RegState::Undef)
1593 .setMIFlag(MachineInstr::FrameSetup);
1594 }
1595
1596 // If this is x86-64 and the Red Zone is not disabled, if we are a leaf
1597 // function, and use up to 128 bytes of stack space, don't have a frame
1598 // pointer, calls, or dynamic alloca then we do not need to adjust the
1599 // stack pointer (we fit in the Red Zone). We also check that we don't
1600 // push and pop from the stack.
1601 if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) &&
1602 !MFI.hasVarSizedObjects() && // No dynamic alloca.
1603 !MFI.adjustsStack() && // No calls.
1604 !EmitStackProbeCall && // No stack probes.
1605 !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop.
1606 !MF.shouldSplitStack()) { // Regular stack
1607 uint64_t MinSize =
1608 X86FI->getCalleeSavedFrameSize() - X86FI->getTCReturnAddrDelta();
1609 if (HasFP) MinSize += SlotSize;
1610 X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0);
1611 StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0);
1612 MFI.setStackSize(StackSize);
1613 }
1614
1615 // Insert stack pointer adjustment for later moving of return addr. Only
1616 // applies to tail call optimized functions where the callee argument stack
1617 // size is bigger than the callers.
1618 if (TailCallArgReserveSize != 0) {
1619 BuildStackAdjustment(MBB, MBBI, DL, -(int)TailCallArgReserveSize,
1620 /*InEpilogue=*/false)
1621 .setMIFlag(MachineInstr::FrameSetup);
1622 }
1623
1624 // Mapping for machine moves:
1625 //
1626 // DST: VirtualFP AND
1627 // SRC: VirtualFP => DW_CFA_def_cfa_offset
1628 // ELSE => DW_CFA_def_cfa
1629 //
1630 // SRC: VirtualFP AND
1631 // DST: Register => DW_CFA_def_cfa_register
1632 //
1633 // ELSE
1634 // OFFSET < 0 => DW_CFA_offset_extended_sf
1635 // REG < 64 => DW_CFA_offset + Reg
1636 // ELSE => DW_CFA_offset_extended
1637
1638 uint64_t NumBytes = 0;
1639 int stackGrowth = -SlotSize;
1640
1641 // Find the funclet establisher parameter
1642 Register Establisher = X86::NoRegister;
1643 if (IsClrFunclet)
1644 Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX;
1645 else if (IsFunclet)
1646 Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX;
1647
1648 if (IsWin64Prologue && IsFunclet && !IsClrFunclet) {
1649 // Immediately spill establisher into the home slot.
1650 // The runtime cares about this.
1651 // MOV64mr %rdx, 16(%rsp)
1652 unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
1653 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16)
1654 .addReg(Establisher)
1655 .setMIFlag(MachineInstr::FrameSetup);
1656 MBB.addLiveIn(Establisher);
1657 }
1658
1659 if (HasFP) {
1660 assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved");
1661
1662 // Calculate required stack adjustment.
1663 uint64_t FrameSize = StackSize - SlotSize;
1664 // If required, include space for extra hidden slot for stashing base pointer.
1665 if (X86FI->getRestoreBasePointer())
1666 FrameSize += SlotSize;
1667
1668 NumBytes = FrameSize -
1669 (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1670
1671 // Callee-saved registers are pushed on stack before the stack is realigned.
1672 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
1673 NumBytes = alignTo(NumBytes, MaxAlign);
1674
1675 // Save EBP/RBP into the appropriate stack slot.
1676 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
1677 .addReg(MachineFramePtr, RegState::Kill)
1678 .setMIFlag(MachineInstr::FrameSetup);
1679
1680 if (NeedsDwarfCFI) {
1681 // Mark the place where EBP/RBP was saved.
1682 // Define the current CFA rule to use the provided offset.
1683 assert(StackSize);
1684 BuildCFI(MBB, MBBI, DL,
1685 MCCFIInstruction::cfiDefCfaOffset(nullptr, -2 * stackGrowth),
1686 MachineInstr::FrameSetup);
1687
1688 // Change the rule for the FramePtr to be an "offset" rule.
1689 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1690 BuildCFI(MBB, MBBI, DL,
1691 MCCFIInstruction::createOffset(nullptr, DwarfFramePtr,
1692 2 * stackGrowth),
1693 MachineInstr::FrameSetup);
1694 }
1695
1696 if (NeedsWinCFI) {
1697 HasWinCFI = true;
1698 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1699 .addImm(FramePtr)
1700 .setMIFlag(MachineInstr::FrameSetup);
1701 }
1702
1703 if (!IsFunclet) {
1704 if (X86FI->hasSwiftAsyncContext()) {
1705 const auto &Attrs = MF.getFunction().getAttributes();
1706
1707 // Before we update the live frame pointer we have to ensure there's a
1708 // valid (or null) asynchronous context in its slot just before FP in
1709 // the frame record, so store it now.
1710 if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) {
1711 // We have an initial context in r14, store it just before the frame
1712 // pointer.
1713 MBB.addLiveIn(X86::R14);
1714 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1715 .addReg(X86::R14)
1716 .setMIFlag(MachineInstr::FrameSetup);
1717 } else {
1718 // No initial context, store null so that there's no pointer that
1719 // could be misused.
1720 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64i8))
1721 .addImm(0)
1722 .setMIFlag(MachineInstr::FrameSetup);
1723 }
1724
1725 if (NeedsWinCFI) {
1726 HasWinCFI = true;
1727 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1728 .addImm(X86::R14)
1729 .setMIFlag(MachineInstr::FrameSetup);
1730 }
1731
1732 BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr)
1733 .addUse(X86::RSP)
1734 .addImm(1)
1735 .addUse(X86::NoRegister)
1736 .addImm(8)
1737 .addUse(X86::NoRegister)
1738 .setMIFlag(MachineInstr::FrameSetup);
1739 BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64ri8), X86::RSP)
1740 .addUse(X86::RSP)
1741 .addImm(8)
1742 .setMIFlag(MachineInstr::FrameSetup);
1743 }
1744
1745 if (!IsWin64Prologue && !IsFunclet) {
1746 // Update EBP with the new base value.
1747 if (!X86FI->hasSwiftAsyncContext())
1748 BuildMI(MBB, MBBI, DL,
1749 TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr),
1750 FramePtr)
1751 .addReg(StackPtr)
1752 .setMIFlag(MachineInstr::FrameSetup);
1753
1754 if (SaveArgs && !Fn.arg_empty()) {
1755 ArrayRef<MCPhysReg> GPRs =
1756 get64BitArgumentGPRs(Fn.getCallingConv(), STI);
1757 unsigned arg_size = Fn.arg_size();
1758 unsigned RI = 0;
1759 int64_t SaveSize = 0;
1760
1761 if (Fn.hasStructRetAttr()) {
1762 GPRs = GPRs.drop_front(1);
1763 arg_size--;
1764 }
1765
1766 for (MCPhysReg Reg : GPRs) {
1767 if (++RI > arg_size)
1768 break;
1769
1770 SaveSize += SlotSize;
1771
1772 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1773 .addReg(Reg)
1774 .setMIFlag(MachineInstr::FrameSetup);
1775 }
1776
1777 // Realign the stack. PUSHes are the most space efficient.
1778 while (SaveSize % getStackAlignment()) {
1779 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1780 .addReg(GPRs.front())
1781 .setMIFlag(MachineInstr::FrameSetup);
1782
1783 SaveSize += SlotSize;
1784 }
1785
1786 //dlg StackSize -= SaveSize;
1787 //dlg MFI.setStackSize(StackSize);
1788 X86FI->setSaveArgSize(SaveSize);
1789 }
1790
1791 if (NeedsDwarfCFI) {
1792 // Mark effective beginning of when frame pointer becomes valid.
1793 // Define the current CFA to use the EBP/RBP register.
1794 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
1795 BuildCFI(
1796 MBB, MBBI, DL,
1797 MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr),
1798 MachineInstr::FrameSetup);
1799 }
1800
1801 if (NeedsWinFPO) {
1802 // .cv_fpo_setframe $FramePtr
1803 HasWinCFI = true;
1804 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
1805 .addImm(FramePtr)
1806 .addImm(0)
1807 .setMIFlag(MachineInstr::FrameSetup);
1808 }
1809 }
1810 }
1811 } else {
1812 assert(!IsFunclet && "funclets without FPs not yet implemented");
1813 NumBytes = StackSize -
1814 (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize);
1815 }
1816
1817 // Update the offset adjustment, which is mainly used by codeview to translate
1818 // from ESP to VFRAME relative local variable offsets.
1819 if (!IsFunclet) {
1820 if (HasFP && TRI->hasStackRealignment(MF))
1821 MFI.setOffsetAdjustment(-NumBytes);
1822 else
1823 MFI.setOffsetAdjustment(-StackSize);
1824 }
1825
1826 // For EH funclets, only allocate enough space for outgoing calls. Save the
1827 // NumBytes value that we would've used for the parent frame.
1828 unsigned ParentFrameNumBytes = NumBytes;
1829 if (IsFunclet)
1830 NumBytes = getWinEHFuncletFrameSize(MF);
1831
1832 // Skip the callee-saved push instructions.
1833 bool PushedRegs = false;
1834 int StackOffset = 2 * stackGrowth;
1835
1836 while (MBBI != MBB.end() &&
1837 MBBI->getFlag(MachineInstr::FrameSetup) &&
1838 (MBBI->getOpcode() == X86::PUSH32r ||
1839 MBBI->getOpcode() == X86::PUSH64r)) {
1840 PushedRegs = true;
1841 Register Reg = MBBI->getOperand(0).getReg();
1842 ++MBBI;
1843
1844 if (!HasFP && NeedsDwarfCFI) {
1845 // Mark callee-saved push instruction.
1846 // Define the current CFA rule to use the provided offset.
1847 assert(StackSize);
1848 BuildCFI(MBB, MBBI, DL,
1849 MCCFIInstruction::cfiDefCfaOffset(nullptr, -StackOffset),
1850 MachineInstr::FrameSetup);
1851 StackOffset += stackGrowth;
1852 }
1853
1854 if (NeedsWinCFI) {
1855 HasWinCFI = true;
1856 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg))
1857 .addImm(Reg)
1858 .setMIFlag(MachineInstr::FrameSetup);
1859 }
1860 }
1861
1862 // Realign stack after we pushed callee-saved registers (so that we'll be
1863 // able to calculate their offsets from the frame pointer).
1864 // Don't do this for Win64, it needs to realign the stack after the prologue.
1865 if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF)) {
1866 assert(HasFP && "There should be a frame pointer if stack is realigned.");
1867 BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign);
1868
1869 if (NeedsWinCFI) {
1870 HasWinCFI = true;
1871 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlign))
1872 .addImm(MaxAlign)
1873 .setMIFlag(MachineInstr::FrameSetup);
1874 }
1875 }
1876
1877 // If there is an SUB32ri of ESP immediately before this instruction, merge
1878 // the two. This can be the case when tail call elimination is enabled and
1879 // the callee has more arguments then the caller.
1880 NumBytes -= mergeSPUpdates(MBB, MBBI, true);
1881
1882 // Adjust stack pointer: ESP -= numbytes.
1883
1884 // Windows and cygwin/mingw require a prologue helper routine when allocating
1885 // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw
1886 // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the
1887 // stack and adjust the stack pointer in one go. The 64-bit version of
1888 // __chkstk is only responsible for probing the stack. The 64-bit prologue is
1889 // responsible for adjusting the stack pointer. Touching the stack at 4K
1890 // increments is necessary to ensure that the guard pages used by the OS
1891 // virtual memory manager are allocated in correct sequence.
1892 uint64_t AlignedNumBytes = NumBytes;
1893 if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF))
1894 AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign);
1895 if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) {
1896 assert(!X86FI->getUsesRedZone() &&
1897 "The Red Zone is not accounted for in stack probes");
1898
1899 // Check whether EAX is livein for this block.
1900 bool isEAXAlive = isEAXLiveIn(MBB);
1901
1902 if (isEAXAlive) {
1903 if (Is64Bit) {
1904 // Save RAX
1905 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r))
1906 .addReg(X86::RAX, RegState::Kill)
1907 .setMIFlag(MachineInstr::FrameSetup);
1908 } else {
1909 // Save EAX
1910 BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r))
1911 .addReg(X86::EAX, RegState::Kill)
1912 .setMIFlag(MachineInstr::FrameSetup);
1913 }
1914 }
1915
1916 if (Is64Bit) {
1917 // Handle the 64-bit Windows ABI case where we need to call __chkstk.
1918 // Function prologue is responsible for adjusting the stack pointer.
1919 int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes;
1920 BuildMI(MBB, MBBI, DL, TII.get(getMOVriOpcode(Is64Bit, Alloc)), X86::RAX)
1921 .addImm(Alloc)
1922 .setMIFlag(MachineInstr::FrameSetup);
1923 } else {
1924 // Allocate NumBytes-4 bytes on stack in case of isEAXAlive.
1925 // We'll also use 4 already allocated bytes for EAX.
1926 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
1927 .addImm(isEAXAlive ? NumBytes - 4 : NumBytes)
1928 .setMIFlag(MachineInstr::FrameSetup);
1929 }
1930
1931 // Call __chkstk, __chkstk_ms, or __alloca.
1932 emitStackProbe(MF, MBB, MBBI, DL, true);
1933
1934 if (isEAXAlive) {
1935 // Restore RAX/EAX
1936 MachineInstr *MI;
1937 if (Is64Bit)
1938 MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV64rm), X86::RAX),
1939 StackPtr, false, NumBytes - 8);
1940 else
1941 MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX),
1942 StackPtr, false, NumBytes - 4);
1943 MI->setFlag(MachineInstr::FrameSetup);
1944 MBB.insert(MBBI, MI);
1945 }
1946 } else if (NumBytes) {
1947 emitSPUpdate(MBB, MBBI, DL, -(int64_t)NumBytes, /*InEpilogue=*/false);
1948 }
1949
1950 if (NeedsWinCFI && NumBytes) {
1951 HasWinCFI = true;
1952 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc))
1953 .addImm(NumBytes)
1954 .setMIFlag(MachineInstr::FrameSetup);
1955 }
1956
1957 int SEHFrameOffset = 0;
1958 unsigned SPOrEstablisher;
1959 if (IsFunclet) {
1960 if (IsClrFunclet) {
1961 // The establisher parameter passed to a CLR funclet is actually a pointer
1962 // to the (mostly empty) frame of its nearest enclosing funclet; we have
1963 // to find the root function establisher frame by loading the PSPSym from
1964 // the intermediate frame.
1965 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
1966 MachinePointerInfo NoInfo;
1967 MBB.addLiveIn(Establisher);
1968 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher),
1969 Establisher, false, PSPSlotOffset)
1970 .addMemOperand(MF.getMachineMemOperand(
1971 NoInfo, MachineMemOperand::MOLoad, SlotSize, Align(SlotSize)));
1972 ;
1973 // Save the root establisher back into the current funclet's (mostly
1974 // empty) frame, in case a sub-funclet or the GC needs it.
1975 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr,
1976 false, PSPSlotOffset)
1977 .addReg(Establisher)
1978 .addMemOperand(MF.getMachineMemOperand(
1979 NoInfo,
1980 MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
1981 SlotSize, Align(SlotSize)));
1982 }
1983 SPOrEstablisher = Establisher;
1984 } else {
1985 SPOrEstablisher = StackPtr;
1986 }
1987
1988 if (IsWin64Prologue && HasFP) {
1989 // Set RBP to a small fixed offset from RSP. In the funclet case, we base
1990 // this calculation on the incoming establisher, which holds the value of
1991 // RSP from the parent frame at the end of the prologue.
1992 SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes);
1993 if (SEHFrameOffset)
1994 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr),
1995 SPOrEstablisher, false, SEHFrameOffset);
1996 else
1997 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr)
1998 .addReg(SPOrEstablisher);
1999
2000 // If this is not a funclet, emit the CFI describing our frame pointer.
2001 if (NeedsWinCFI && !IsFunclet) {
2002 assert(!NeedsWinFPO && "this setframe incompatible with FPO data");
2003 HasWinCFI = true;
2004 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame))
2005 .addImm(FramePtr)
2006 .addImm(SEHFrameOffset)
2007 .setMIFlag(MachineInstr::FrameSetup);
2008 if (isAsynchronousEHPersonality(Personality))
2009 MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset;
2010 }
2011 } else if (IsFunclet && STI.is32Bit()) {
2012 // Reset EBP / ESI to something good for funclets.
2013 MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL);
2014 // If we're a catch funclet, we can be returned to via catchret. Save ESP
2015 // into the registration node so that the runtime will restore it for us.
2016 if (!MBB.isCleanupFuncletEntry()) {
2017 assert(Personality == EHPersonality::MSVC_CXX);
2018 Register FrameReg;
2019 int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex;
2020 int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed();
2021 // ESP is the first field, so no extra displacement is needed.
2022 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg,
2023 false, EHRegOffset)
2024 .addReg(X86::ESP);
2025 }
2026 }
2027
2028 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) {
2029 const MachineInstr &FrameInstr = *MBBI;
2030 ++MBBI;
2031
2032 if (NeedsWinCFI) {
2033 int FI;
2034 if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) {
2035 if (X86::FR64RegClass.contains(Reg)) {
2036 int Offset;
2037 Register IgnoredFrameReg;
2038 if (IsWin64Prologue && IsFunclet)
2039 Offset = getWin64EHFrameIndexRef(MF, FI, IgnoredFrameReg);
2040 else
2041 Offset =
2042 getFrameIndexReference(MF, FI, IgnoredFrameReg).getFixed() +
2043 SEHFrameOffset;
2044
2045 HasWinCFI = true;
2046 assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data");
2047 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM))
2048 .addImm(Reg)
2049 .addImm(Offset)
2050 .setMIFlag(MachineInstr::FrameSetup);
2051 }
2052 }
2053 }
2054 }
2055
2056 if (NeedsWinCFI && HasWinCFI)
2057 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue))
2058 .setMIFlag(MachineInstr::FrameSetup);
2059
2060 if (FnHasClrFunclet && !IsFunclet) {
2061 // Save the so-called Initial-SP (i.e. the value of the stack pointer
2062 // immediately after the prolog) into the PSPSlot so that funclets
2063 // and the GC can recover it.
2064 unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF);
2065 auto PSPInfo = MachinePointerInfo::getFixedStack(
2066 MF, MF.getWinEHFuncInfo()->PSPSymFrameIdx);
2067 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false,
2068 PSPSlotOffset)
2069 .addReg(StackPtr)
2070 .addMemOperand(MF.getMachineMemOperand(
2071 PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile,
2072 SlotSize, Align(SlotSize)));
2073 }
2074
2075 // Realign stack after we spilled callee-saved registers (so that we'll be
2076 // able to calculate their offsets from the frame pointer).
2077 // Win64 requires aligning the stack after the prologue.
2078 if (IsWin64Prologue && TRI->hasStackRealignment(MF)) {
2079 assert(HasFP && "There should be a frame pointer if stack is realigned.");
2080 BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign);
2081 }
2082
2083 // We already dealt with stack realignment and funclets above.
2084 if (IsFunclet && STI.is32Bit())
2085 return;
2086
2087 // If we need a base pointer, set it up here. It's whatever the value
2088 // of the stack pointer is at this point. Any variable size objects
2089 // will be allocated after this, so we can still use the base pointer
2090 // to reference locals.
2091 if (TRI->hasBasePointer(MF)) {
2092 // Update the base pointer with the current stack pointer.
2093 unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr;
2094 BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr)
2095 .addReg(SPOrEstablisher)
2096 .setMIFlag(MachineInstr::FrameSetup);
2097 if (X86FI->getRestoreBasePointer()) {
2098 // Stash value of base pointer. Saving RSP instead of EBP shortens
2099 // dependence chain. Used by SjLj EH.
2100 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2101 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)),
2102 FramePtr, true, X86FI->getRestoreBasePointerOffset())
2103 .addReg(SPOrEstablisher)
2104 .setMIFlag(MachineInstr::FrameSetup);
2105 }
2106
2107 if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) {
2108 // Stash the value of the frame pointer relative to the base pointer for
2109 // Win32 EH. This supports Win32 EH, which does the inverse of the above:
2110 // it recovers the frame pointer from the base pointer rather than the
2111 // other way around.
2112 unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr;
2113 Register UsedReg;
2114 int Offset =
2115 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg)
2116 .getFixed();
2117 assert(UsedReg == BasePtr);
2118 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset)
2119 .addReg(FramePtr)
2120 .setMIFlag(MachineInstr::FrameSetup);
2121 }
2122 }
2123
2124 if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) {
2125 // Mark end of stack pointer adjustment.
2126 if (!HasFP && NumBytes) {
2127 // Define the current CFA rule to use the provided offset.
2128 assert(StackSize);
2129 BuildCFI(
2130 MBB, MBBI, DL,
2131 MCCFIInstruction::cfiDefCfaOffset(nullptr, StackSize - stackGrowth),
2132 MachineInstr::FrameSetup);
2133 }
2134
2135 // Emit DWARF info specifying the offsets of the callee-saved registers.
2136 emitCalleeSavedFrameMoves(MBB, MBBI, DL, true);
2137 }
2138
2139 // X86 Interrupt handling function cannot assume anything about the direction
2140 // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction
2141 // in each prologue of interrupt handler function.
2142 //
2143 // FIXME: Create "cld" instruction only in these cases:
2144 // 1. The interrupt handling function uses any of the "rep" instructions.
2145 // 2. Interrupt handling function calls another function.
2146 //
2147 if (Fn.getCallingConv() == CallingConv::X86_INTR)
2148 BuildMI(MBB, MBBI, DL, TII.get(X86::CLD))
2149 .setMIFlag(MachineInstr::FrameSetup);
2150
2151 // At this point we know if the function has WinCFI or not.
2152 MF.setHasWinCFI(HasWinCFI);
2153 }
2154
canUseLEAForSPInEpilogue(const MachineFunction & MF) const2155 bool X86FrameLowering::canUseLEAForSPInEpilogue(
2156 const MachineFunction &MF) const {
2157 // We can't use LEA instructions for adjusting the stack pointer if we don't
2158 // have a frame pointer in the Win64 ABI. Only ADD instructions may be used
2159 // to deallocate the stack.
2160 // This means that we can use LEA for SP in two situations:
2161 // 1. We *aren't* using the Win64 ABI which means we are free to use LEA.
2162 // 2. We *have* a frame pointer which means we are permitted to use LEA.
2163 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF);
2164 }
2165
isFuncletReturnInstr(MachineInstr & MI)2166 static bool isFuncletReturnInstr(MachineInstr &MI) {
2167 switch (MI.getOpcode()) {
2168 case X86::CATCHRET:
2169 case X86::CLEANUPRET:
2170 return true;
2171 default:
2172 return false;
2173 }
2174 llvm_unreachable("impossible");
2175 }
2176
2177 // CLR funclets use a special "Previous Stack Pointer Symbol" slot on the
2178 // stack. It holds a pointer to the bottom of the root function frame. The
2179 // establisher frame pointer passed to a nested funclet may point to the
2180 // (mostly empty) frame of its parent funclet, but it will need to find
2181 // the frame of the root function to access locals. To facilitate this,
2182 // every funclet copies the pointer to the bottom of the root function
2183 // frame into a PSPSym slot in its own (mostly empty) stack frame. Using the
2184 // same offset for the PSPSym in the root function frame that's used in the
2185 // funclets' frames allows each funclet to dynamically accept any ancestor
2186 // frame as its establisher argument (the runtime doesn't guarantee the
2187 // immediate parent for some reason lost to history), and also allows the GC,
2188 // which uses the PSPSym for some bookkeeping, to find it in any funclet's
2189 // frame with only a single offset reported for the entire method.
2190 unsigned
getPSPSlotOffsetFromSP(const MachineFunction & MF) const2191 X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const {
2192 const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo();
2193 Register SPReg;
2194 int Offset = getFrameIndexReferencePreferSP(MF, Info.PSPSymFrameIdx, SPReg,
2195 /*IgnoreSPUpdates*/ true)
2196 .getFixed();
2197 assert(Offset >= 0 && SPReg == TRI->getStackRegister());
2198 return static_cast<unsigned>(Offset);
2199 }
2200
2201 unsigned
getWinEHFuncletFrameSize(const MachineFunction & MF) const2202 X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
2203 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2204 // This is the size of the pushed CSRs.
2205 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2206 // This is the size of callee saved XMMs.
2207 const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2208 unsigned XMMSize = WinEHXMMSlotInfo.size() *
2209 TRI->getSpillSize(X86::VR128RegClass);
2210 // This is the amount of stack a funclet needs to allocate.
2211 unsigned UsedSize;
2212 EHPersonality Personality =
2213 classifyEHPersonality(MF.getFunction().getPersonalityFn());
2214 if (Personality == EHPersonality::CoreCLR) {
2215 // CLR funclets need to hold enough space to include the PSPSym, at the
2216 // same offset from the stack pointer (immediately after the prolog) as it
2217 // resides at in the main function.
2218 UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize;
2219 } else {
2220 // Other funclets just need enough stack for outgoing call arguments.
2221 UsedSize = MF.getFrameInfo().getMaxCallFrameSize();
2222 }
2223 // RBP is not included in the callee saved register block. After pushing RBP,
2224 // everything is 16 byte aligned. Everything we allocate before an outgoing
2225 // call must also be 16 byte aligned.
2226 unsigned FrameSizeMinusRBP = alignTo(CSSize + UsedSize, getStackAlign());
2227 // Subtract out the size of the callee saved registers. This is how much stack
2228 // each funclet will allocate.
2229 return FrameSizeMinusRBP + XMMSize - CSSize;
2230 }
2231
isTailCallOpcode(unsigned Opc)2232 static bool isTailCallOpcode(unsigned Opc) {
2233 return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
2234 Opc == X86::TCRETURNmi ||
2235 Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 ||
2236 Opc == X86::TCRETURNmi64;
2237 }
2238
emitEpilogue(MachineFunction & MF,MachineBasicBlock & MBB) const2239 void X86FrameLowering::emitEpilogue(MachineFunction &MF,
2240 MachineBasicBlock &MBB) const {
2241 const MachineFrameInfo &MFI = MF.getFrameInfo();
2242 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2243 MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator();
2244 MachineBasicBlock::iterator MBBI = Terminator;
2245 DebugLoc DL;
2246 if (MBBI != MBB.end())
2247 DL = MBBI->getDebugLoc();
2248 // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit.
2249 const bool Is64BitILP32 = STI.isTarget64BitILP32();
2250 Register FramePtr = TRI->getFrameRegister(MF);
2251 Register MachineFramePtr =
2252 Is64BitILP32 ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr;
2253
2254 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2255 bool NeedsWin64CFI =
2256 IsWin64Prologue && MF.getFunction().needsUnwindTableEntry();
2257 bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(*MBBI);
2258
2259 // Get the number of bytes to allocate from the FrameInfo.
2260 uint64_t StackSize = MFI.getStackSize();
2261 uint64_t MaxAlign = calculateMaxStackAlign(MF);
2262 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2263 unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta();
2264 bool HasFP = hasFP(MF);
2265 uint64_t NumBytes = 0;
2266
2267 bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() &&
2268 !MF.getTarget().getTargetTriple().isOSWindows()) &&
2269 MF.needsFrameMoves();
2270
2271 if (IsFunclet) {
2272 assert(HasFP && "EH funclets without FP not yet implemented");
2273 NumBytes = getWinEHFuncletFrameSize(MF);
2274 } else if (HasFP) {
2275 // Calculate required stack adjustment.
2276 uint64_t FrameSize = StackSize - SlotSize;
2277 NumBytes = FrameSize - CSSize - TailCallArgReserveSize;
2278
2279 // Callee-saved registers were pushed on stack before the stack was
2280 // realigned.
2281 if (TRI->hasStackRealignment(MF) && !IsWin64Prologue)
2282 NumBytes = alignTo(FrameSize, MaxAlign);
2283 } else {
2284 NumBytes = StackSize - CSSize - TailCallArgReserveSize;
2285 }
2286 uint64_t SEHStackAllocAmt = NumBytes;
2287
2288 // AfterPop is the position to insert .cfi_restore.
2289 MachineBasicBlock::iterator AfterPop = MBBI;
2290 if (HasFP) {
2291 if (X86FI->hasSwiftAsyncContext()) {
2292 // Discard the context.
2293 int Offset = 16 + mergeSPUpdates(MBB, MBBI, true);
2294 emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue*/true);
2295 }
2296
2297 if (X86FI->getSaveArgSize()) {
2298 // LEAVE is effectively mov rbp,rsp; pop rbp
2299 BuildMI(MBB, MBBI, DL, TII.get(X86::LEAVE64))
2300 .setMIFlag(MachineInstr::FrameDestroy);
2301 } else {
2302 // Pop EBP.
2303 BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r),
2304 MachineFramePtr)
2305 .setMIFlag(MachineInstr::FrameDestroy);
2306 }
2307
2308 // We need to reset FP to its untagged state on return. Bit 60 is currently
2309 // used to show the presence of an extended frame.
2310 if (X86FI->hasSwiftAsyncContext()) {
2311 BuildMI(MBB, MBBI, DL, TII.get(X86::BTR64ri8),
2312 MachineFramePtr)
2313 .addUse(MachineFramePtr)
2314 .addImm(60)
2315 .setMIFlag(MachineInstr::FrameDestroy);
2316 }
2317
2318 if (NeedsDwarfCFI) {
2319 unsigned DwarfStackPtr =
2320 TRI->getDwarfRegNum(Is64Bit ? X86::RSP : X86::ESP, true);
2321 BuildCFI(MBB, MBBI, DL,
2322 MCCFIInstruction::cfiDefCfa(nullptr, DwarfStackPtr, SlotSize),
2323 MachineInstr::FrameDestroy);
2324 if (!MBB.succ_empty() && !MBB.isReturnBlock()) {
2325 unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true);
2326 BuildCFI(MBB, AfterPop, DL,
2327 MCCFIInstruction::createRestore(nullptr, DwarfFramePtr),
2328 MachineInstr::FrameDestroy);
2329 --MBBI;
2330 --AfterPop;
2331 }
2332 --MBBI;
2333 }
2334 }
2335
2336 MachineBasicBlock::iterator FirstCSPop = MBBI;
2337 // Skip the callee-saved pop instructions.
2338 while (MBBI != MBB.begin()) {
2339 MachineBasicBlock::iterator PI = std::prev(MBBI);
2340 unsigned Opc = PI->getOpcode();
2341
2342 if (Opc != X86::DBG_VALUE && !PI->isTerminator()) {
2343 if ((Opc != X86::POP32r || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2344 (Opc != X86::POP64r || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2345 (Opc != X86::LEAVE64 || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2346 (Opc != X86::BTR64ri8 || !PI->getFlag(MachineInstr::FrameDestroy)) &&
2347 (Opc != X86::ADD64ri8 || !PI->getFlag(MachineInstr::FrameDestroy)))
2348 break;
2349 FirstCSPop = PI;
2350 }
2351
2352 --MBBI;
2353 }
2354 MBBI = FirstCSPop;
2355
2356 if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET)
2357 emitCatchRetReturnValue(MBB, FirstCSPop, &*Terminator);
2358
2359 if (MBBI != MBB.end())
2360 DL = MBBI->getDebugLoc();
2361 // If there is an ADD32ri or SUB32ri of ESP immediately before this
2362 // instruction, merge the two instructions.
2363 if (NumBytes || MFI.hasVarSizedObjects())
2364 NumBytes += mergeSPUpdates(MBB, MBBI, true);
2365
2366 // If dynamic alloca is used, then reset esp to point to the last callee-saved
2367 // slot before popping them off! Same applies for the case, when stack was
2368 // realigned. Don't do this if this was a funclet epilogue, since the funclets
2369 // will not do realignment or dynamic stack allocation.
2370 if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) &&
2371 !IsFunclet) {
2372 if (TRI->hasStackRealignment(MF))
2373 MBBI = FirstCSPop;
2374 unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt);
2375 uint64_t LEAAmount =
2376 IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize;
2377
2378 if (X86FI->hasSwiftAsyncContext())
2379 LEAAmount -= 16;
2380
2381 // There are only two legal forms of epilogue:
2382 // - add SEHAllocationSize, %rsp
2383 // - lea SEHAllocationSize(%FramePtr), %rsp
2384 //
2385 // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence.
2386 // However, we may use this sequence if we have a frame pointer because the
2387 // effects of the prologue can safely be undone.
2388 if (LEAAmount != 0) {
2389 unsigned Opc = getLEArOpcode(Uses64BitFramePtr);
2390 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr),
2391 FramePtr, false, LEAAmount);
2392 --MBBI;
2393 } else {
2394 unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr);
2395 BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr)
2396 .addReg(FramePtr);
2397 --MBBI;
2398 }
2399 } else if (NumBytes) {
2400 // Adjust stack pointer back: ESP += numbytes.
2401 emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true);
2402 if (!HasFP && NeedsDwarfCFI) {
2403 // Define the current CFA rule to use the provided offset.
2404 BuildCFI(MBB, MBBI, DL,
2405 MCCFIInstruction::cfiDefCfaOffset(
2406 nullptr, CSSize + TailCallArgReserveSize + SlotSize),
2407 MachineInstr::FrameDestroy);
2408 }
2409 --MBBI;
2410 }
2411
2412 // Windows unwinder will not invoke function's exception handler if IP is
2413 // either in prologue or in epilogue. This behavior causes a problem when a
2414 // call immediately precedes an epilogue, because the return address points
2415 // into the epilogue. To cope with that, we insert an epilogue marker here,
2416 // then replace it with a 'nop' if it ends up immediately after a CALL in the
2417 // final emitted code.
2418 if (NeedsWin64CFI && MF.hasWinCFI())
2419 BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue));
2420
2421 if (!HasFP && NeedsDwarfCFI) {
2422 MBBI = FirstCSPop;
2423 int64_t Offset = -CSSize - SlotSize;
2424 // Mark callee-saved pop instruction.
2425 // Define the current CFA rule to use the provided offset.
2426 while (MBBI != MBB.end()) {
2427 MachineBasicBlock::iterator PI = MBBI;
2428 unsigned Opc = PI->getOpcode();
2429 ++MBBI;
2430 if (Opc == X86::POP32r || Opc == X86::POP64r) {
2431 Offset += SlotSize;
2432 BuildCFI(MBB, MBBI, DL,
2433 MCCFIInstruction::cfiDefCfaOffset(nullptr, -Offset),
2434 MachineInstr::FrameDestroy);
2435 }
2436 }
2437 }
2438
2439 // Emit DWARF info specifying the restores of the callee-saved registers.
2440 // For epilogue with return inside or being other block without successor,
2441 // no need to generate .cfi_restore for callee-saved registers.
2442 if (NeedsDwarfCFI && !MBB.succ_empty())
2443 emitCalleeSavedFrameMoves(MBB, AfterPop, DL, false);
2444
2445 if (Terminator == MBB.end() || !isTailCallOpcode(Terminator->getOpcode())) {
2446 // Add the return addr area delta back since we are not tail calling.
2447 int Offset = -1 * X86FI->getTCReturnAddrDelta();
2448 assert(Offset >= 0 && "TCDelta should never be positive");
2449 if (Offset) {
2450 // Check for possible merge with preceding ADD instruction.
2451 Offset += mergeSPUpdates(MBB, Terminator, true);
2452 emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true);
2453 }
2454 }
2455
2456 // Emit tilerelease for AMX kernel.
2457 if (X86FI->hasVirtualTileReg())
2458 BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
2459 }
2460
getFrameIndexReference(const MachineFunction & MF,int FI,Register & FrameReg) const2461 StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
2462 int FI,
2463 Register &FrameReg) const {
2464 const MachineFrameInfo &MFI = MF.getFrameInfo();
2465
2466 bool IsFixed = MFI.isFixedObjectIndex(FI);
2467 // We can't calculate offset from frame pointer if the stack is realigned,
2468 // so enforce usage of stack/base pointer. The base pointer is used when we
2469 // have dynamic allocas in addition to dynamic realignment.
2470 if (TRI->hasBasePointer(MF))
2471 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister();
2472 else if (TRI->hasStackRealignment(MF))
2473 FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister();
2474 else
2475 FrameReg = TRI->getFrameRegister(MF);
2476
2477 // Offset will hold the offset from the stack pointer at function entry to the
2478 // object.
2479 // We need to factor in additional offsets applied during the prologue to the
2480 // frame, base, and stack pointer depending on which is used.
2481 int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
2482 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2483 unsigned CSSize = X86FI->getCalleeSavedFrameSize();
2484 uint64_t StackSize = MFI.getStackSize();
2485 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
2486 int64_t FPDelta = 0;
2487
2488 // In an x86 interrupt, remove the offset we added to account for the return
2489 // address from any stack object allocated in the caller's frame. Interrupts
2490 // do not have a standard return address. Fixed objects in the current frame,
2491 // such as SSE register spills, should not get this treatment.
2492 if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR &&
2493 Offset >= 0) {
2494 Offset += getOffsetOfLocalArea();
2495 }
2496
2497 if (IsWin64Prologue) {
2498 assert(!MFI.hasCalls() || (StackSize % 16) == 8);
2499
2500 // Calculate required stack adjustment.
2501 uint64_t FrameSize = StackSize - SlotSize;
2502 // If required, include space for extra hidden slot for stashing base pointer.
2503 if (X86FI->getRestoreBasePointer())
2504 FrameSize += SlotSize;
2505 uint64_t NumBytes = FrameSize - CSSize;
2506
2507 uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes);
2508 if (FI && FI == X86FI->getFAIndex())
2509 return StackOffset::getFixed(-SEHFrameOffset);
2510
2511 // FPDelta is the offset from the "traditional" FP location of the old base
2512 // pointer followed by return address and the location required by the
2513 // restricted Win64 prologue.
2514 // Add FPDelta to all offsets below that go through the frame pointer.
2515 FPDelta = FrameSize - SEHFrameOffset;
2516 assert((!MFI.hasCalls() || (FPDelta % 16) == 0) &&
2517 "FPDelta isn't aligned per the Win64 ABI!");
2518 }
2519
2520 if (FI >= 0)
2521 Offset -= X86FI->getSaveArgSize();
2522
2523 if (FrameReg == TRI->getFramePtr()) {
2524 // Skip saved EBP/RBP
2525 Offset += SlotSize;
2526
2527 // Account for restricted Windows prologue.
2528 Offset += FPDelta;
2529
2530 // Skip the RETADDR move area
2531 int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2532 if (TailCallReturnAddrDelta < 0)
2533 Offset -= TailCallReturnAddrDelta;
2534
2535 return StackOffset::getFixed(Offset);
2536 }
2537
2538 // FrameReg is either the stack pointer or a base pointer. But the base is
2539 // located at the end of the statically known StackSize so the distinction
2540 // doesn't really matter.
2541 if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF))
2542 assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize)));
2543 return StackOffset::getFixed(Offset + StackSize);
2544 }
2545
getWin64EHFrameIndexRef(const MachineFunction & MF,int FI,Register & FrameReg) const2546 int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, int FI,
2547 Register &FrameReg) const {
2548 const MachineFrameInfo &MFI = MF.getFrameInfo();
2549 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2550 const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2551 const auto it = WinEHXMMSlotInfo.find(FI);
2552
2553 if (it == WinEHXMMSlotInfo.end())
2554 return getFrameIndexReference(MF, FI, FrameReg).getFixed();
2555
2556 FrameReg = TRI->getStackRegister();
2557 return alignDown(MFI.getMaxCallFrameSize(), getStackAlign().value()) +
2558 it->second;
2559 }
2560
2561 StackOffset
getFrameIndexReferenceSP(const MachineFunction & MF,int FI,Register & FrameReg,int Adjustment) const2562 X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, int FI,
2563 Register &FrameReg,
2564 int Adjustment) const {
2565 const MachineFrameInfo &MFI = MF.getFrameInfo();
2566 FrameReg = TRI->getStackRegister();
2567 return StackOffset::getFixed(MFI.getObjectOffset(FI) -
2568 getOffsetOfLocalArea() + Adjustment);
2569 }
2570
2571 StackOffset
getFrameIndexReferencePreferSP(const MachineFunction & MF,int FI,Register & FrameReg,bool IgnoreSPUpdates) const2572 X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF,
2573 int FI, Register &FrameReg,
2574 bool IgnoreSPUpdates) const {
2575
2576 const MachineFrameInfo &MFI = MF.getFrameInfo();
2577 // Does not include any dynamic realign.
2578 const uint64_t StackSize = MFI.getStackSize();
2579 // LLVM arranges the stack as follows:
2580 // ...
2581 // ARG2
2582 // ARG1
2583 // RETADDR
2584 // PUSH RBP <-- RBP points here
2585 // PUSH CSRs
2586 // ~~~~~~~ <-- possible stack realignment (non-win64)
2587 // ...
2588 // STACK OBJECTS
2589 // ... <-- RSP after prologue points here
2590 // ~~~~~~~ <-- possible stack realignment (win64)
2591 //
2592 // if (hasVarSizedObjects()):
2593 // ... <-- "base pointer" (ESI/RBX) points here
2594 // DYNAMIC ALLOCAS
2595 // ... <-- RSP points here
2596 //
2597 // Case 1: In the simple case of no stack realignment and no dynamic
2598 // allocas, both "fixed" stack objects (arguments and CSRs) are addressable
2599 // with fixed offsets from RSP.
2600 //
2601 // Case 2: In the case of stack realignment with no dynamic allocas, fixed
2602 // stack objects are addressed with RBP and regular stack objects with RSP.
2603 //
2604 // Case 3: In the case of dynamic allocas and stack realignment, RSP is used
2605 // to address stack arguments for outgoing calls and nothing else. The "base
2606 // pointer" points to local variables, and RBP points to fixed objects.
2607 //
2608 // In cases 2 and 3, we can only answer for non-fixed stack objects, and the
2609 // answer we give is relative to the SP after the prologue, and not the
2610 // SP in the middle of the function.
2611
2612 if (MFI.isFixedObjectIndex(FI) && TRI->hasStackRealignment(MF) &&
2613 !STI.isTargetWin64())
2614 return getFrameIndexReference(MF, FI, FrameReg);
2615
2616 // If !hasReservedCallFrame the function might have SP adjustement in the
2617 // body. So, even though the offset is statically known, it depends on where
2618 // we are in the function.
2619 if (!IgnoreSPUpdates && !hasReservedCallFrame(MF))
2620 return getFrameIndexReference(MF, FI, FrameReg);
2621
2622 // We don't handle tail calls, and shouldn't be seeing them either.
2623 assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 &&
2624 "we don't handle this case!");
2625
2626 // This is how the math works out:
2627 //
2628 // %rsp grows (i.e. gets lower) left to right. Each box below is
2629 // one word (eight bytes). Obj0 is the stack slot we're trying to
2630 // get to.
2631 //
2632 // ----------------------------------
2633 // | BP | Obj0 | Obj1 | ... | ObjN |
2634 // ----------------------------------
2635 // ^ ^ ^ ^
2636 // A B C E
2637 //
2638 // A is the incoming stack pointer.
2639 // (B - A) is the local area offset (-8 for x86-64) [1]
2640 // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2]
2641 //
2642 // |(E - B)| is the StackSize (absolute value, positive). For a
2643 // stack that grown down, this works out to be (B - E). [3]
2644 //
2645 // E is also the value of %rsp after stack has been set up, and we
2646 // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
2647 // (C - E) == (C - A) - (B - A) + (B - E)
2648 // { Using [1], [2] and [3] above }
2649 // == getObjectOffset - LocalAreaOffset + StackSize
2650
2651 return getFrameIndexReferenceSP(MF, FI, FrameReg, StackSize);
2652 }
2653
assignCalleeSavedSpillSlots(MachineFunction & MF,const TargetRegisterInfo * TRI,std::vector<CalleeSavedInfo> & CSI) const2654 bool X86FrameLowering::assignCalleeSavedSpillSlots(
2655 MachineFunction &MF, const TargetRegisterInfo *TRI,
2656 std::vector<CalleeSavedInfo> &CSI) const {
2657 MachineFrameInfo &MFI = MF.getFrameInfo();
2658 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
2659
2660 unsigned CalleeSavedFrameSize = 0;
2661 unsigned XMMCalleeSavedFrameSize = 0;
2662 auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo();
2663 int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta();
2664
2665 int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta();
2666
2667 if (TailCallReturnAddrDelta < 0) {
2668 // create RETURNADDR area
2669 // arg
2670 // arg
2671 // RETADDR
2672 // { ...
2673 // RETADDR area
2674 // ...
2675 // }
2676 // [EBP]
2677 MFI.CreateFixedObject(-TailCallReturnAddrDelta,
2678 TailCallReturnAddrDelta - SlotSize, true);
2679 }
2680
2681 // Spill the BasePtr if it's used.
2682 if (this->TRI->hasBasePointer(MF)) {
2683 // Allocate a spill slot for EBP if we have a base pointer and EH funclets.
2684 if (MF.hasEHFunclets()) {
2685 int FI = MFI.CreateSpillStackObject(SlotSize, Align(SlotSize));
2686 X86FI->setHasSEHFramePtrSave(true);
2687 X86FI->setSEHFramePtrSaveIndex(FI);
2688 }
2689 }
2690
2691 if (hasFP(MF)) {
2692 // emitPrologue always spills frame register the first thing.
2693 SpillSlotOffset -= SlotSize;
2694 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2695
2696 // The async context lives directly before the frame pointer, and we
2697 // allocate a second slot to preserve stack alignment.
2698 if (X86FI->hasSwiftAsyncContext()) {
2699 SpillSlotOffset -= SlotSize;
2700 MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2701 SpillSlotOffset -= SlotSize;
2702 }
2703
2704 // Since emitPrologue and emitEpilogue will handle spilling and restoring of
2705 // the frame register, we can delete it from CSI list and not have to worry
2706 // about avoiding it later.
2707 Register FPReg = TRI->getFrameRegister(MF);
2708 for (unsigned i = 0; i < CSI.size(); ++i) {
2709 if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) {
2710 CSI.erase(CSI.begin() + i);
2711 break;
2712 }
2713 }
2714 }
2715
2716 // Assign slots for GPRs. It increases frame size.
2717 for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
2718 Register Reg = I.getReg();
2719
2720 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
2721 continue;
2722
2723 SpillSlotOffset -= SlotSize;
2724 CalleeSavedFrameSize += SlotSize;
2725
2726 int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset);
2727 I.setFrameIdx(SlotIndex);
2728 }
2729
2730 X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize);
2731 MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize);
2732
2733 // Assign slots for XMMs.
2734 for (CalleeSavedInfo &I : llvm::reverse(CSI)) {
2735 Register Reg = I.getReg();
2736 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
2737 continue;
2738
2739 // If this is k-register make sure we lookup via the largest legal type.
2740 MVT VT = MVT::Other;
2741 if (X86::VK16RegClass.contains(Reg))
2742 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2743
2744 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2745 unsigned Size = TRI->getSpillSize(*RC);
2746 Align Alignment = TRI->getSpillAlign(*RC);
2747 // ensure alignment
2748 assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86");
2749 SpillSlotOffset = -alignTo(-SpillSlotOffset, Alignment);
2750
2751 // spill into slot
2752 SpillSlotOffset -= Size;
2753 int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset);
2754 I.setFrameIdx(SlotIndex);
2755 MFI.ensureMaxAlignment(Alignment);
2756
2757 // Save the start offset and size of XMM in stack frame for funclets.
2758 if (X86::VR128RegClass.contains(Reg)) {
2759 WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize;
2760 XMMCalleeSavedFrameSize += Size;
2761 }
2762 }
2763
2764 return true;
2765 }
2766
spillCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,ArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const2767 bool X86FrameLowering::spillCalleeSavedRegisters(
2768 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2769 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2770 DebugLoc DL = MBB.findDebugLoc(MI);
2771
2772 // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI
2773 // for us, and there are no XMM CSRs on Win32.
2774 if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows())
2775 return true;
2776
2777 // Push GPRs. It increases frame size.
2778 const MachineFunction &MF = *MBB.getParent();
2779 unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
2780 for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
2781 Register Reg = I.getReg();
2782
2783 if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg))
2784 continue;
2785
2786 const MachineRegisterInfo &MRI = MF.getRegInfo();
2787 bool isLiveIn = MRI.isLiveIn(Reg);
2788 if (!isLiveIn)
2789 MBB.addLiveIn(Reg);
2790
2791 // Decide whether we can add a kill flag to the use.
2792 bool CanKill = !isLiveIn;
2793 // Check if any subregister is live-in
2794 if (CanKill) {
2795 for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg) {
2796 if (MRI.isLiveIn(*AReg)) {
2797 CanKill = false;
2798 break;
2799 }
2800 }
2801 }
2802
2803 // Do not set a kill flag on values that are also marked as live-in. This
2804 // happens with the @llvm-returnaddress intrinsic and with arguments
2805 // passed in callee saved registers.
2806 // Omitting the kill flags is conservatively correct even if the live-in
2807 // is not used after all.
2808 BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, getKillRegState(CanKill))
2809 .setMIFlag(MachineInstr::FrameSetup);
2810 }
2811
2812 // Make XMM regs spilled. X86 does not have ability of push/pop XMM.
2813 // It can be done by spilling XMMs to stack frame.
2814 for (const CalleeSavedInfo &I : llvm::reverse(CSI)) {
2815 Register Reg = I.getReg();
2816 if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg))
2817 continue;
2818
2819 // If this is k-register make sure we lookup via the largest legal type.
2820 MVT VT = MVT::Other;
2821 if (X86::VK16RegClass.contains(Reg))
2822 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2823
2824 // Add the callee-saved register as live-in. It's killed at the spill.
2825 MBB.addLiveIn(Reg);
2826 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2827
2828 TII.storeRegToStackSlot(MBB, MI, Reg, true, I.getFrameIdx(), RC, TRI,
2829 Register());
2830 --MI;
2831 MI->setFlag(MachineInstr::FrameSetup);
2832 ++MI;
2833 }
2834
2835 return true;
2836 }
2837
emitCatchRetReturnValue(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,MachineInstr * CatchRet) const2838 void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB,
2839 MachineBasicBlock::iterator MBBI,
2840 MachineInstr *CatchRet) const {
2841 // SEH shouldn't use catchret.
2842 assert(!isAsynchronousEHPersonality(classifyEHPersonality(
2843 MBB.getParent()->getFunction().getPersonalityFn())) &&
2844 "SEH should not use CATCHRET");
2845 const DebugLoc &DL = CatchRet->getDebugLoc();
2846 MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(0).getMBB();
2847
2848 // Fill EAX/RAX with the address of the target block.
2849 if (STI.is64Bit()) {
2850 // LEA64r CatchRetTarget(%rip), %rax
2851 BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), X86::RAX)
2852 .addReg(X86::RIP)
2853 .addImm(0)
2854 .addReg(0)
2855 .addMBB(CatchRetTarget)
2856 .addReg(0);
2857 } else {
2858 // MOV32ri $CatchRetTarget, %eax
2859 BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX)
2860 .addMBB(CatchRetTarget);
2861 }
2862
2863 // Record that we've taken the address of CatchRetTarget and no longer just
2864 // reference it in a terminator.
2865 CatchRetTarget->setMachineBlockAddressTaken();
2866 }
2867
restoreCalleeSavedRegisters(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,MutableArrayRef<CalleeSavedInfo> CSI,const TargetRegisterInfo * TRI) const2868 bool X86FrameLowering::restoreCalleeSavedRegisters(
2869 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2870 MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
2871 if (CSI.empty())
2872 return false;
2873
2874 if (MI != MBB.end() && isFuncletReturnInstr(*MI) && STI.isOSWindows()) {
2875 // Don't restore CSRs in 32-bit EH funclets. Matches
2876 // spillCalleeSavedRegisters.
2877 if (STI.is32Bit())
2878 return true;
2879 // Don't restore CSRs before an SEH catchret. SEH except blocks do not form
2880 // funclets. emitEpilogue transforms these to normal jumps.
2881 if (MI->getOpcode() == X86::CATCHRET) {
2882 const Function &F = MBB.getParent()->getFunction();
2883 bool IsSEH = isAsynchronousEHPersonality(
2884 classifyEHPersonality(F.getPersonalityFn()));
2885 if (IsSEH)
2886 return true;
2887 }
2888 }
2889
2890 DebugLoc DL = MBB.findDebugLoc(MI);
2891
2892 // Reload XMMs from stack frame.
2893 for (const CalleeSavedInfo &I : CSI) {
2894 Register Reg = I.getReg();
2895 if (X86::GR64RegClass.contains(Reg) ||
2896 X86::GR32RegClass.contains(Reg))
2897 continue;
2898
2899 // If this is k-register make sure we lookup via the largest legal type.
2900 MVT VT = MVT::Other;
2901 if (X86::VK16RegClass.contains(Reg))
2902 VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1;
2903
2904 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2905 TII.loadRegFromStackSlot(MBB, MI, Reg, I.getFrameIdx(), RC, TRI,
2906 Register());
2907 }
2908
2909 // POP GPRs.
2910 unsigned Opc = STI.is64Bit() ? X86::POP64r : X86::POP32r;
2911 for (const CalleeSavedInfo &I : CSI) {
2912 Register Reg = I.getReg();
2913 if (!X86::GR64RegClass.contains(Reg) &&
2914 !X86::GR32RegClass.contains(Reg))
2915 continue;
2916
2917 BuildMI(MBB, MI, DL, TII.get(Opc), Reg)
2918 .setMIFlag(MachineInstr::FrameDestroy);
2919 }
2920 return true;
2921 }
2922
determineCalleeSaves(MachineFunction & MF,BitVector & SavedRegs,RegScavenger * RS) const2923 void X86FrameLowering::determineCalleeSaves(MachineFunction &MF,
2924 BitVector &SavedRegs,
2925 RegScavenger *RS) const {
2926 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2927
2928 // Spill the BasePtr if it's used.
2929 if (TRI->hasBasePointer(MF)){
2930 Register BasePtr = TRI->getBaseRegister();
2931 if (STI.isTarget64BitILP32())
2932 BasePtr = getX86SubSuperRegister(BasePtr, 64);
2933 SavedRegs.set(BasePtr);
2934 }
2935 }
2936
2937 static bool
HasNestArgument(const MachineFunction * MF)2938 HasNestArgument(const MachineFunction *MF) {
2939 const Function &F = MF->getFunction();
2940 for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end();
2941 I != E; I++) {
2942 if (I->hasNestAttr() && !I->use_empty())
2943 return true;
2944 }
2945 return false;
2946 }
2947
2948 /// GetScratchRegister - Get a temp register for performing work in the
2949 /// segmented stack and the Erlang/HiPE stack prologue. Depending on platform
2950 /// and the properties of the function either one or two registers will be
2951 /// needed. Set primary to true for the first register, false for the second.
2952 static unsigned
GetScratchRegister(bool Is64Bit,bool IsLP64,const MachineFunction & MF,bool Primary)2953 GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Primary) {
2954 CallingConv::ID CallingConvention = MF.getFunction().getCallingConv();
2955
2956 // Erlang stuff.
2957 if (CallingConvention == CallingConv::HiPE) {
2958 if (Is64Bit)
2959 return Primary ? X86::R14 : X86::R13;
2960 else
2961 return Primary ? X86::EBX : X86::EDI;
2962 }
2963
2964 if (Is64Bit) {
2965 if (IsLP64)
2966 return Primary ? X86::R11 : X86::R12;
2967 else
2968 return Primary ? X86::R11D : X86::R12D;
2969 }
2970
2971 bool IsNested = HasNestArgument(&MF);
2972
2973 if (CallingConvention == CallingConv::X86_FastCall ||
2974 CallingConvention == CallingConv::Fast ||
2975 CallingConvention == CallingConv::Tail) {
2976 if (IsNested)
2977 report_fatal_error("Segmented stacks does not support fastcall with "
2978 "nested function.");
2979 return Primary ? X86::EAX : X86::ECX;
2980 }
2981 if (IsNested)
2982 return Primary ? X86::EDX : X86::EAX;
2983 return Primary ? X86::ECX : X86::EAX;
2984 }
2985
2986 // The stack limit in the TCB is set to this many bytes above the actual stack
2987 // limit.
2988 static const uint64_t kSplitStackAvailable = 256;
2989
adjustForSegmentedStacks(MachineFunction & MF,MachineBasicBlock & PrologueMBB) const2990 void X86FrameLowering::adjustForSegmentedStacks(
2991 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
2992 MachineFrameInfo &MFI = MF.getFrameInfo();
2993 uint64_t StackSize;
2994 unsigned TlsReg, TlsOffset;
2995 DebugLoc DL;
2996
2997 // To support shrink-wrapping we would need to insert the new blocks
2998 // at the right place and update the branches to PrologueMBB.
2999 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3000
3001 unsigned ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3002 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3003 "Scratch register is live-in");
3004
3005 if (MF.getFunction().isVarArg())
3006 report_fatal_error("Segmented stacks do not support vararg functions.");
3007 if (!STI.isTargetLinux() && !STI.isTargetDarwin() && !STI.isTargetWin32() &&
3008 !STI.isTargetWin64() && !STI.isTargetFreeBSD() &&
3009 !STI.isTargetDragonFly())
3010 report_fatal_error("Segmented stacks not supported on this platform.");
3011
3012 // Eventually StackSize will be calculated by a link-time pass; which will
3013 // also decide whether checking code needs to be injected into this particular
3014 // prologue.
3015 StackSize = MFI.getStackSize();
3016
3017 if (!MFI.needsSplitStackProlog())
3018 return;
3019
3020 MachineBasicBlock *allocMBB = MF.CreateMachineBasicBlock();
3021 MachineBasicBlock *checkMBB = MF.CreateMachineBasicBlock();
3022 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3023 bool IsNested = false;
3024
3025 // We need to know if the function has a nest argument only in 64 bit mode.
3026 if (Is64Bit)
3027 IsNested = HasNestArgument(&MF);
3028
3029 // The MOV R10, RAX needs to be in a different block, since the RET we emit in
3030 // allocMBB needs to be last (terminating) instruction.
3031
3032 for (const auto &LI : PrologueMBB.liveins()) {
3033 allocMBB->addLiveIn(LI);
3034 checkMBB->addLiveIn(LI);
3035 }
3036
3037 if (IsNested)
3038 allocMBB->addLiveIn(IsLP64 ? X86::R10 : X86::R10D);
3039
3040 MF.push_front(allocMBB);
3041 MF.push_front(checkMBB);
3042
3043 // When the frame size is less than 256 we just compare the stack
3044 // boundary directly to the value of the stack pointer, per gcc.
3045 bool CompareStackPointer = StackSize < kSplitStackAvailable;
3046
3047 // Read the limit off the current stacklet off the stack_guard location.
3048 if (Is64Bit) {
3049 if (STI.isTargetLinux()) {
3050 TlsReg = X86::FS;
3051 TlsOffset = IsLP64 ? 0x70 : 0x40;
3052 } else if (STI.isTargetDarwin()) {
3053 TlsReg = X86::GS;
3054 TlsOffset = 0x60 + 90*8; // See pthread_machdep.h. Steal TLS slot 90.
3055 } else if (STI.isTargetWin64()) {
3056 TlsReg = X86::GS;
3057 TlsOffset = 0x28; // pvArbitrary, reserved for application use
3058 } else if (STI.isTargetFreeBSD()) {
3059 TlsReg = X86::FS;
3060 TlsOffset = 0x18;
3061 } else if (STI.isTargetDragonFly()) {
3062 TlsReg = X86::FS;
3063 TlsOffset = 0x20; // use tls_tcb.tcb_segstack
3064 } else {
3065 report_fatal_error("Segmented stacks not supported on this platform.");
3066 }
3067
3068 if (CompareStackPointer)
3069 ScratchReg = IsLP64 ? X86::RSP : X86::ESP;
3070 else
3071 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::LEA64r : X86::LEA64_32r), ScratchReg).addReg(X86::RSP)
3072 .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
3073
3074 BuildMI(checkMBB, DL, TII.get(IsLP64 ? X86::CMP64rm : X86::CMP32rm)).addReg(ScratchReg)
3075 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg);
3076 } else {
3077 if (STI.isTargetLinux()) {
3078 TlsReg = X86::GS;
3079 TlsOffset = 0x30;
3080 } else if (STI.isTargetDarwin()) {
3081 TlsReg = X86::GS;
3082 TlsOffset = 0x48 + 90*4;
3083 } else if (STI.isTargetWin32()) {
3084 TlsReg = X86::FS;
3085 TlsOffset = 0x14; // pvArbitrary, reserved for application use
3086 } else if (STI.isTargetDragonFly()) {
3087 TlsReg = X86::FS;
3088 TlsOffset = 0x10; // use tls_tcb.tcb_segstack
3089 } else if (STI.isTargetFreeBSD()) {
3090 report_fatal_error("Segmented stacks not supported on FreeBSD i386.");
3091 } else {
3092 report_fatal_error("Segmented stacks not supported on this platform.");
3093 }
3094
3095 if (CompareStackPointer)
3096 ScratchReg = X86::ESP;
3097 else
3098 BuildMI(checkMBB, DL, TII.get(X86::LEA32r), ScratchReg).addReg(X86::ESP)
3099 .addImm(1).addReg(0).addImm(-StackSize).addReg(0);
3100
3101 if (STI.isTargetLinux() || STI.isTargetWin32() || STI.isTargetWin64() ||
3102 STI.isTargetDragonFly()) {
3103 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm)).addReg(ScratchReg)
3104 .addReg(0).addImm(0).addReg(0).addImm(TlsOffset).addReg(TlsReg);
3105 } else if (STI.isTargetDarwin()) {
3106
3107 // TlsOffset doesn't fit into a mod r/m byte so we need an extra register.
3108 unsigned ScratchReg2;
3109 bool SaveScratch2;
3110 if (CompareStackPointer) {
3111 // The primary scratch register is available for holding the TLS offset.
3112 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3113 SaveScratch2 = false;
3114 } else {
3115 // Need to use a second register to hold the TLS offset
3116 ScratchReg2 = GetScratchRegister(Is64Bit, IsLP64, MF, false);
3117
3118 // Unfortunately, with fastcc the second scratch register may hold an
3119 // argument.
3120 SaveScratch2 = MF.getRegInfo().isLiveIn(ScratchReg2);
3121 }
3122
3123 // If Scratch2 is live-in then it needs to be saved.
3124 assert((!MF.getRegInfo().isLiveIn(ScratchReg2) || SaveScratch2) &&
3125 "Scratch register is live-in and not saved");
3126
3127 if (SaveScratch2)
3128 BuildMI(checkMBB, DL, TII.get(X86::PUSH32r))
3129 .addReg(ScratchReg2, RegState::Kill);
3130
3131 BuildMI(checkMBB, DL, TII.get(X86::MOV32ri), ScratchReg2)
3132 .addImm(TlsOffset);
3133 BuildMI(checkMBB, DL, TII.get(X86::CMP32rm))
3134 .addReg(ScratchReg)
3135 .addReg(ScratchReg2).addImm(1).addReg(0)
3136 .addImm(0)
3137 .addReg(TlsReg);
3138
3139 if (SaveScratch2)
3140 BuildMI(checkMBB, DL, TII.get(X86::POP32r), ScratchReg2);
3141 }
3142 }
3143
3144 // This jump is taken if SP >= (Stacklet Limit + Stack Space required).
3145 // It jumps to normal execution of the function body.
3146 BuildMI(checkMBB, DL, TII.get(X86::JCC_1)).addMBB(&PrologueMBB).addImm(X86::COND_A);
3147
3148 // On 32 bit we first push the arguments size and then the frame size. On 64
3149 // bit, we pass the stack frame size in r10 and the argument size in r11.
3150 if (Is64Bit) {
3151 // Functions with nested arguments use R10, so it needs to be saved across
3152 // the call to _morestack
3153
3154 const unsigned RegAX = IsLP64 ? X86::RAX : X86::EAX;
3155 const unsigned Reg10 = IsLP64 ? X86::R10 : X86::R10D;
3156 const unsigned Reg11 = IsLP64 ? X86::R11 : X86::R11D;
3157 const unsigned MOVrr = IsLP64 ? X86::MOV64rr : X86::MOV32rr;
3158
3159 if (IsNested)
3160 BuildMI(allocMBB, DL, TII.get(MOVrr), RegAX).addReg(Reg10);
3161
3162 BuildMI(allocMBB, DL, TII.get(getMOVriOpcode(IsLP64, StackSize)), Reg10)
3163 .addImm(StackSize);
3164 BuildMI(allocMBB, DL,
3165 TII.get(getMOVriOpcode(IsLP64, X86FI->getArgumentStackSize())),
3166 Reg11)
3167 .addImm(X86FI->getArgumentStackSize());
3168 } else {
3169 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
3170 .addImm(X86FI->getArgumentStackSize());
3171 BuildMI(allocMBB, DL, TII.get(X86::PUSHi32))
3172 .addImm(StackSize);
3173 }
3174
3175 // __morestack is in libgcc
3176 if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) {
3177 // Under the large code model, we cannot assume that __morestack lives
3178 // within 2^31 bytes of the call site, so we cannot use pc-relative
3179 // addressing. We cannot perform the call via a temporary register,
3180 // as the rax register may be used to store the static chain, and all
3181 // other suitable registers may be either callee-save or used for
3182 // parameter passing. We cannot use the stack at this point either
3183 // because __morestack manipulates the stack directly.
3184 //
3185 // To avoid these issues, perform an indirect call via a read-only memory
3186 // location containing the address.
3187 //
3188 // This solution is not perfect, as it assumes that the .rodata section
3189 // is laid out within 2^31 bytes of each function body, but this seems
3190 // to be sufficient for JIT.
3191 // FIXME: Add retpoline support and remove the error here..
3192 if (STI.useIndirectThunkCalls())
3193 report_fatal_error("Emitting morestack calls on 64-bit with the large "
3194 "code model and thunks not yet implemented.");
3195 BuildMI(allocMBB, DL, TII.get(X86::CALL64m))
3196 .addReg(X86::RIP)
3197 .addImm(0)
3198 .addReg(0)
3199 .addExternalSymbol("__morestack_addr")
3200 .addReg(0);
3201 } else {
3202 if (Is64Bit)
3203 BuildMI(allocMBB, DL, TII.get(X86::CALL64pcrel32))
3204 .addExternalSymbol("__morestack");
3205 else
3206 BuildMI(allocMBB, DL, TII.get(X86::CALLpcrel32))
3207 .addExternalSymbol("__morestack");
3208 }
3209
3210 if (IsNested)
3211 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET_RESTORE_R10));
3212 else
3213 BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET));
3214
3215 allocMBB->addSuccessor(&PrologueMBB);
3216
3217 checkMBB->addSuccessor(allocMBB, BranchProbability::getZero());
3218 checkMBB->addSuccessor(&PrologueMBB, BranchProbability::getOne());
3219
3220 #ifdef EXPENSIVE_CHECKS
3221 MF.verify();
3222 #endif
3223 }
3224
3225 /// Lookup an ERTS parameter in the !hipe.literals named metadata node.
3226 /// HiPE provides Erlang Runtime System-internal parameters, such as PCB offsets
3227 /// to fields it needs, through a named metadata node "hipe.literals" containing
3228 /// name-value pairs.
getHiPELiteral(NamedMDNode * HiPELiteralsMD,const StringRef LiteralName)3229 static unsigned getHiPELiteral(
3230 NamedMDNode *HiPELiteralsMD, const StringRef LiteralName) {
3231 for (int i = 0, e = HiPELiteralsMD->getNumOperands(); i != e; ++i) {
3232 MDNode *Node = HiPELiteralsMD->getOperand(i);
3233 if (Node->getNumOperands() != 2) continue;
3234 MDString *NodeName = dyn_cast<MDString>(Node->getOperand(0));
3235 ValueAsMetadata *NodeVal = dyn_cast<ValueAsMetadata>(Node->getOperand(1));
3236 if (!NodeName || !NodeVal) continue;
3237 ConstantInt *ValConst = dyn_cast_or_null<ConstantInt>(NodeVal->getValue());
3238 if (ValConst && NodeName->getString() == LiteralName) {
3239 return ValConst->getZExtValue();
3240 }
3241 }
3242
3243 report_fatal_error("HiPE literal " + LiteralName
3244 + " required but not provided");
3245 }
3246
3247 // Return true if there are no non-ehpad successors to MBB and there are no
3248 // non-meta instructions between MBBI and MBB.end().
blockEndIsUnreachable(const MachineBasicBlock & MBB,MachineBasicBlock::const_iterator MBBI)3249 static bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
3250 MachineBasicBlock::const_iterator MBBI) {
3251 return llvm::all_of(
3252 MBB.successors(),
3253 [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
3254 std::all_of(MBBI, MBB.end(), [](const MachineInstr &MI) {
3255 return MI.isMetaInstruction();
3256 });
3257 }
3258
3259 /// Erlang programs may need a special prologue to handle the stack size they
3260 /// might need at runtime. That is because Erlang/OTP does not implement a C
3261 /// stack but uses a custom implementation of hybrid stack/heap architecture.
3262 /// (for more information see Eric Stenman's Ph.D. thesis:
3263 /// http://publications.uu.se/uu/fulltext/nbn_se_uu_diva-2688.pdf)
3264 ///
3265 /// CheckStack:
3266 /// temp0 = sp - MaxStack
3267 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
3268 /// OldStart:
3269 /// ...
3270 /// IncStack:
3271 /// call inc_stack # doubles the stack space
3272 /// temp0 = sp - MaxStack
3273 /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart
adjustForHiPEPrologue(MachineFunction & MF,MachineBasicBlock & PrologueMBB) const3274 void X86FrameLowering::adjustForHiPEPrologue(
3275 MachineFunction &MF, MachineBasicBlock &PrologueMBB) const {
3276 MachineFrameInfo &MFI = MF.getFrameInfo();
3277 DebugLoc DL;
3278
3279 // To support shrink-wrapping we would need to insert the new blocks
3280 // at the right place and update the branches to PrologueMBB.
3281 assert(&(*MF.begin()) == &PrologueMBB && "Shrink-wrapping not supported yet");
3282
3283 // HiPE-specific values
3284 NamedMDNode *HiPELiteralsMD = MF.getMMI().getModule()
3285 ->getNamedMetadata("hipe.literals");
3286 if (!HiPELiteralsMD)
3287 report_fatal_error(
3288 "Can't generate HiPE prologue without runtime parameters");
3289 const unsigned HipeLeafWords
3290 = getHiPELiteral(HiPELiteralsMD,
3291 Is64Bit ? "AMD64_LEAF_WORDS" : "X86_LEAF_WORDS");
3292 const unsigned CCRegisteredArgs = Is64Bit ? 6 : 5;
3293 const unsigned Guaranteed = HipeLeafWords * SlotSize;
3294 unsigned CallerStkArity = MF.getFunction().arg_size() > CCRegisteredArgs ?
3295 MF.getFunction().arg_size() - CCRegisteredArgs : 0;
3296 unsigned MaxStack = MFI.getStackSize() + CallerStkArity*SlotSize + SlotSize;
3297
3298 assert(STI.isTargetLinux() &&
3299 "HiPE prologue is only supported on Linux operating systems.");
3300
3301 // Compute the largest caller's frame that is needed to fit the callees'
3302 // frames. This 'MaxStack' is computed from:
3303 //
3304 // a) the fixed frame size, which is the space needed for all spilled temps,
3305 // b) outgoing on-stack parameter areas, and
3306 // c) the minimum stack space this function needs to make available for the
3307 // functions it calls (a tunable ABI property).
3308 if (MFI.hasCalls()) {
3309 unsigned MoreStackForCalls = 0;
3310
3311 for (auto &MBB : MF) {
3312 for (auto &MI : MBB) {
3313 if (!MI.isCall())
3314 continue;
3315
3316 // Get callee operand.
3317 const MachineOperand &MO = MI.getOperand(0);
3318
3319 // Only take account of global function calls (no closures etc.).
3320 if (!MO.isGlobal())
3321 continue;
3322
3323 const Function *F = dyn_cast<Function>(MO.getGlobal());
3324 if (!F)
3325 continue;
3326
3327 // Do not update 'MaxStack' for primitive and built-in functions
3328 // (encoded with names either starting with "erlang."/"bif_" or not
3329 // having a ".", such as a simple <Module>.<Function>.<Arity>, or an
3330 // "_", such as the BIF "suspend_0") as they are executed on another
3331 // stack.
3332 if (F->getName().contains("erlang.") || F->getName().contains("bif_") ||
3333 F->getName().find_first_of("._") == StringRef::npos)
3334 continue;
3335
3336 unsigned CalleeStkArity =
3337 F->arg_size() > CCRegisteredArgs ? F->arg_size()-CCRegisteredArgs : 0;
3338 if (HipeLeafWords - 1 > CalleeStkArity)
3339 MoreStackForCalls = std::max(MoreStackForCalls,
3340 (HipeLeafWords - 1 - CalleeStkArity) * SlotSize);
3341 }
3342 }
3343 MaxStack += MoreStackForCalls;
3344 }
3345
3346 // If the stack frame needed is larger than the guaranteed then runtime checks
3347 // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue.
3348 if (MaxStack > Guaranteed) {
3349 MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock();
3350 MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock();
3351
3352 for (const auto &LI : PrologueMBB.liveins()) {
3353 stackCheckMBB->addLiveIn(LI);
3354 incStackMBB->addLiveIn(LI);
3355 }
3356
3357 MF.push_front(incStackMBB);
3358 MF.push_front(stackCheckMBB);
3359
3360 unsigned ScratchReg, SPReg, PReg, SPLimitOffset;
3361 unsigned LEAop, CMPop, CALLop;
3362 SPLimitOffset = getHiPELiteral(HiPELiteralsMD, "P_NSP_LIMIT");
3363 if (Is64Bit) {
3364 SPReg = X86::RSP;
3365 PReg = X86::RBP;
3366 LEAop = X86::LEA64r;
3367 CMPop = X86::CMP64rm;
3368 CALLop = X86::CALL64pcrel32;
3369 } else {
3370 SPReg = X86::ESP;
3371 PReg = X86::EBP;
3372 LEAop = X86::LEA32r;
3373 CMPop = X86::CMP32rm;
3374 CALLop = X86::CALLpcrel32;
3375 }
3376
3377 ScratchReg = GetScratchRegister(Is64Bit, IsLP64, MF, true);
3378 assert(!MF.getRegInfo().isLiveIn(ScratchReg) &&
3379 "HiPE prologue scratch register is live-in");
3380
3381 // Create new MBB for StackCheck:
3382 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(LEAop), ScratchReg),
3383 SPReg, false, -MaxStack);
3384 // SPLimitOffset is in a fixed heap location (pointed by BP).
3385 addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop))
3386 .addReg(ScratchReg), PReg, false, SPLimitOffset);
3387 BuildMI(stackCheckMBB, DL, TII.get(X86::JCC_1)).addMBB(&PrologueMBB).addImm(X86::COND_AE);
3388
3389 // Create new MBB for IncStack:
3390 BuildMI(incStackMBB, DL, TII.get(CALLop)).
3391 addExternalSymbol("inc_stack_0");
3392 addRegOffset(BuildMI(incStackMBB, DL, TII.get(LEAop), ScratchReg),
3393 SPReg, false, -MaxStack);
3394 addRegOffset(BuildMI(incStackMBB, DL, TII.get(CMPop))
3395 .addReg(ScratchReg), PReg, false, SPLimitOffset);
3396 BuildMI(incStackMBB, DL, TII.get(X86::JCC_1)).addMBB(incStackMBB).addImm(X86::COND_LE);
3397
3398 stackCheckMBB->addSuccessor(&PrologueMBB, {99, 100});
3399 stackCheckMBB->addSuccessor(incStackMBB, {1, 100});
3400 incStackMBB->addSuccessor(&PrologueMBB, {99, 100});
3401 incStackMBB->addSuccessor(incStackMBB, {1, 100});
3402 }
3403 #ifdef EXPENSIVE_CHECKS
3404 MF.verify();
3405 #endif
3406 }
3407
adjustStackWithPops(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,int Offset) const3408 bool X86FrameLowering::adjustStackWithPops(MachineBasicBlock &MBB,
3409 MachineBasicBlock::iterator MBBI,
3410 const DebugLoc &DL,
3411 int Offset) const {
3412 if (Offset <= 0)
3413 return false;
3414
3415 if (Offset % SlotSize)
3416 return false;
3417
3418 int NumPops = Offset / SlotSize;
3419 // This is only worth it if we have at most 2 pops.
3420 if (NumPops != 1 && NumPops != 2)
3421 return false;
3422
3423 // Handle only the trivial case where the adjustment directly follows
3424 // a call. This is the most common one, anyway.
3425 if (MBBI == MBB.begin())
3426 return false;
3427 MachineBasicBlock::iterator Prev = std::prev(MBBI);
3428 if (!Prev->isCall() || !Prev->getOperand(1).isRegMask())
3429 return false;
3430
3431 unsigned Regs[2];
3432 unsigned FoundRegs = 0;
3433
3434 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3435 const MachineOperand &RegMask = Prev->getOperand(1);
3436
3437 auto &RegClass =
3438 Is64Bit ? X86::GR64_NOREX_NOSPRegClass : X86::GR32_NOREX_NOSPRegClass;
3439 // Try to find up to NumPops free registers.
3440 for (auto Candidate : RegClass) {
3441 // Poor man's liveness:
3442 // Since we're immediately after a call, any register that is clobbered
3443 // by the call and not defined by it can be considered dead.
3444 if (!RegMask.clobbersPhysReg(Candidate))
3445 continue;
3446
3447 // Don't clobber reserved registers
3448 if (MRI.isReserved(Candidate))
3449 continue;
3450
3451 bool IsDef = false;
3452 for (const MachineOperand &MO : Prev->implicit_operands()) {
3453 if (MO.isReg() && MO.isDef() &&
3454 TRI->isSuperOrSubRegisterEq(MO.getReg(), Candidate)) {
3455 IsDef = true;
3456 break;
3457 }
3458 }
3459
3460 if (IsDef)
3461 continue;
3462
3463 Regs[FoundRegs++] = Candidate;
3464 if (FoundRegs == (unsigned)NumPops)
3465 break;
3466 }
3467
3468 if (FoundRegs == 0)
3469 return false;
3470
3471 // If we found only one free register, but need two, reuse the same one twice.
3472 while (FoundRegs < (unsigned)NumPops)
3473 Regs[FoundRegs++] = Regs[0];
3474
3475 for (int i = 0; i < NumPops; ++i)
3476 BuildMI(MBB, MBBI, DL,
3477 TII.get(STI.is64Bit() ? X86::POP64r : X86::POP32r), Regs[i]);
3478
3479 return true;
3480 }
3481
3482 MachineBasicBlock::iterator X86FrameLowering::
eliminateCallFramePseudoInstr(MachineFunction & MF,MachineBasicBlock & MBB,MachineBasicBlock::iterator I) const3483 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
3484 MachineBasicBlock::iterator I) const {
3485 bool reserveCallFrame = hasReservedCallFrame(MF);
3486 unsigned Opcode = I->getOpcode();
3487 bool isDestroy = Opcode == TII.getCallFrameDestroyOpcode();
3488 DebugLoc DL = I->getDebugLoc(); // copy DebugLoc as I will be erased.
3489 uint64_t Amount = TII.getFrameSize(*I);
3490 uint64_t InternalAmt = (isDestroy || Amount) ? TII.getFrameAdjustment(*I) : 0;
3491 I = MBB.erase(I);
3492 auto InsertPos = skipDebugInstructionsForward(I, MBB.end());
3493
3494 // Try to avoid emitting dead SP adjustments if the block end is unreachable,
3495 // typically because the function is marked noreturn (abort, throw,
3496 // assert_fail, etc).
3497 if (isDestroy && blockEndIsUnreachable(MBB, I))
3498 return I;
3499
3500 if (!reserveCallFrame) {
3501 // If the stack pointer can be changed after prologue, turn the
3502 // adjcallstackup instruction into a 'sub ESP, <amt>' and the
3503 // adjcallstackdown instruction into 'add ESP, <amt>'
3504
3505 // We need to keep the stack aligned properly. To do this, we round the
3506 // amount of space needed for the outgoing arguments up to the next
3507 // alignment boundary.
3508 Amount = alignTo(Amount, getStackAlign());
3509
3510 const Function &F = MF.getFunction();
3511 bool WindowsCFI = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
3512 bool DwarfCFI = !WindowsCFI && MF.needsFrameMoves();
3513
3514 // If we have any exception handlers in this function, and we adjust
3515 // the SP before calls, we may need to indicate this to the unwinder
3516 // using GNU_ARGS_SIZE. Note that this may be necessary even when
3517 // Amount == 0, because the preceding function may have set a non-0
3518 // GNU_ARGS_SIZE.
3519 // TODO: We don't need to reset this between subsequent functions,
3520 // if it didn't change.
3521 bool HasDwarfEHHandlers = !WindowsCFI && !MF.getLandingPads().empty();
3522
3523 if (HasDwarfEHHandlers && !isDestroy &&
3524 MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences())
3525 BuildCFI(MBB, InsertPos, DL,
3526 MCCFIInstruction::createGnuArgsSize(nullptr, Amount));
3527
3528 if (Amount == 0)
3529 return I;
3530
3531 // Factor out the amount that gets handled inside the sequence
3532 // (Pushes of argument for frame setup, callee pops for frame destroy)
3533 Amount -= InternalAmt;
3534
3535 // TODO: This is needed only if we require precise CFA.
3536 // If this is a callee-pop calling convention, emit a CFA adjust for
3537 // the amount the callee popped.
3538 if (isDestroy && InternalAmt && DwarfCFI && !hasFP(MF))
3539 BuildCFI(MBB, InsertPos, DL,
3540 MCCFIInstruction::createAdjustCfaOffset(nullptr, -InternalAmt));
3541
3542 // Add Amount to SP to destroy a frame, or subtract to setup.
3543 int64_t StackAdjustment = isDestroy ? Amount : -Amount;
3544
3545 if (StackAdjustment) {
3546 // Merge with any previous or following adjustment instruction. Note: the
3547 // instructions merged with here do not have CFI, so their stack
3548 // adjustments do not feed into CfaAdjustment.
3549 StackAdjustment += mergeSPUpdates(MBB, InsertPos, true);
3550 StackAdjustment += mergeSPUpdates(MBB, InsertPos, false);
3551
3552 if (StackAdjustment) {
3553 if (!(F.hasMinSize() &&
3554 adjustStackWithPops(MBB, InsertPos, DL, StackAdjustment)))
3555 BuildStackAdjustment(MBB, InsertPos, DL, StackAdjustment,
3556 /*InEpilogue=*/false);
3557 }
3558 }
3559
3560 if (DwarfCFI && !hasFP(MF)) {
3561 // If we don't have FP, but need to generate unwind information,
3562 // we need to set the correct CFA offset after the stack adjustment.
3563 // How much we adjust the CFA offset depends on whether we're emitting
3564 // CFI only for EH purposes or for debugging. EH only requires the CFA
3565 // offset to be correct at each call site, while for debugging we want
3566 // it to be more precise.
3567
3568 int64_t CfaAdjustment = -StackAdjustment;
3569 // TODO: When not using precise CFA, we also need to adjust for the
3570 // InternalAmt here.
3571 if (CfaAdjustment) {
3572 BuildCFI(MBB, InsertPos, DL,
3573 MCCFIInstruction::createAdjustCfaOffset(nullptr,
3574 CfaAdjustment));
3575 }
3576 }
3577
3578 return I;
3579 }
3580
3581 if (InternalAmt) {
3582 MachineBasicBlock::iterator CI = I;
3583 MachineBasicBlock::iterator B = MBB.begin();
3584 while (CI != B && !std::prev(CI)->isCall())
3585 --CI;
3586 BuildStackAdjustment(MBB, CI, DL, -InternalAmt, /*InEpilogue=*/false);
3587 }
3588
3589 return I;
3590 }
3591
canUseAsPrologue(const MachineBasicBlock & MBB) const3592 bool X86FrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
3593 assert(MBB.getParent() && "Block is not attached to a function!");
3594 const MachineFunction &MF = *MBB.getParent();
3595 if (!MBB.isLiveIn(X86::EFLAGS))
3596 return true;
3597
3598 // If stack probes have to loop inline or call, that will clobber EFLAGS.
3599 // FIXME: we could allow cases that will use emitStackProbeInlineGenericBlock.
3600 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
3601 const X86TargetLowering &TLI = *STI.getTargetLowering();
3602 if (TLI.hasInlineStackProbe(MF) || TLI.hasStackProbeSymbol(MF))
3603 return false;
3604
3605 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3606 return !TRI->hasStackRealignment(MF) && !X86FI->hasSwiftAsyncContext();
3607 }
3608
canUseAsEpilogue(const MachineBasicBlock & MBB) const3609 bool X86FrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
3610 assert(MBB.getParent() && "Block is not attached to a function!");
3611
3612 // Win64 has strict requirements in terms of epilogue and we are
3613 // not taking a chance at messing with them.
3614 // I.e., unless this block is already an exit block, we can't use
3615 // it as an epilogue.
3616 if (STI.isTargetWin64() && !MBB.succ_empty() && !MBB.isReturnBlock())
3617 return false;
3618
3619 // Swift async context epilogue has a BTR instruction that clobbers parts of
3620 // EFLAGS.
3621 const MachineFunction &MF = *MBB.getParent();
3622 if (MF.getInfo<X86MachineFunctionInfo>()->hasSwiftAsyncContext())
3623 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
3624
3625 if (canUseLEAForSPInEpilogue(*MBB.getParent()))
3626 return true;
3627
3628 // If we cannot use LEA to adjust SP, we may need to use ADD, which
3629 // clobbers the EFLAGS. Check that we do not need to preserve it,
3630 // otherwise, conservatively assume this is not
3631 // safe to insert the epilogue here.
3632 return !flagsNeedToBePreservedBeforeTheTerminators(MBB);
3633 }
3634
enableShrinkWrapping(const MachineFunction & MF) const3635 bool X86FrameLowering::enableShrinkWrapping(const MachineFunction &MF) const {
3636 // If we may need to emit frameless compact unwind information, give
3637 // up as this is currently broken: PR25614.
3638 bool CompactUnwind =
3639 MF.getMMI().getContext().getObjectFileInfo()->getCompactUnwindSection() !=
3640 nullptr;
3641 return (MF.getFunction().hasFnAttribute(Attribute::NoUnwind) || hasFP(MF) ||
3642 !CompactUnwind) &&
3643 // The lowering of segmented stack and HiPE only support entry
3644 // blocks as prologue blocks: PR26107. This limitation may be
3645 // lifted if we fix:
3646 // - adjustForSegmentedStacks
3647 // - adjustForHiPEPrologue
3648 MF.getFunction().getCallingConv() != CallingConv::HiPE &&
3649 !MF.shouldSplitStack();
3650 }
3651
restoreWin32EHStackPointers(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,const DebugLoc & DL,bool RestoreSP) const3652 MachineBasicBlock::iterator X86FrameLowering::restoreWin32EHStackPointers(
3653 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
3654 const DebugLoc &DL, bool RestoreSP) const {
3655 assert(STI.isTargetWindowsMSVC() && "funclets only supported in MSVC env");
3656 assert(STI.isTargetWin32() && "EBP/ESI restoration only required on win32");
3657 assert(STI.is32Bit() && !Uses64BitFramePtr &&
3658 "restoring EBP/ESI on non-32-bit target");
3659
3660 MachineFunction &MF = *MBB.getParent();
3661 Register FramePtr = TRI->getFrameRegister(MF);
3662 Register BasePtr = TRI->getBaseRegister();
3663 WinEHFuncInfo &FuncInfo = *MF.getWinEHFuncInfo();
3664 X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
3665 MachineFrameInfo &MFI = MF.getFrameInfo();
3666
3667 // FIXME: Don't set FrameSetup flag in catchret case.
3668
3669 int FI = FuncInfo.EHRegNodeFrameIndex;
3670 int EHRegSize = MFI.getObjectSize(FI);
3671
3672 if (RestoreSP) {
3673 // MOV32rm -EHRegSize(%ebp), %esp
3674 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), X86::ESP),
3675 X86::EBP, true, -EHRegSize)
3676 .setMIFlag(MachineInstr::FrameSetup);
3677 }
3678
3679 Register UsedReg;
3680 int EHRegOffset = getFrameIndexReference(MF, FI, UsedReg).getFixed();
3681 int EndOffset = -EHRegOffset - EHRegSize;
3682 FuncInfo.EHRegNodeEndOffset = EndOffset;
3683
3684 if (UsedReg == FramePtr) {
3685 // ADD $offset, %ebp
3686 unsigned ADDri = getADDriOpcode(false, EndOffset);
3687 BuildMI(MBB, MBBI, DL, TII.get(ADDri), FramePtr)
3688 .addReg(FramePtr)
3689 .addImm(EndOffset)
3690 .setMIFlag(MachineInstr::FrameSetup)
3691 ->getOperand(3)
3692 .setIsDead();
3693 assert(EndOffset >= 0 &&
3694 "end of registration object above normal EBP position!");
3695 } else if (UsedReg == BasePtr) {
3696 // LEA offset(%ebp), %esi
3697 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA32r), BasePtr),
3698 FramePtr, false, EndOffset)
3699 .setMIFlag(MachineInstr::FrameSetup);
3700 // MOV32rm SavedEBPOffset(%esi), %ebp
3701 assert(X86FI->getHasSEHFramePtrSave());
3702 int Offset =
3703 getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg)
3704 .getFixed();
3705 assert(UsedReg == BasePtr);
3706 addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32rm), FramePtr),
3707 UsedReg, true, Offset)
3708 .setMIFlag(MachineInstr::FrameSetup);
3709 } else {
3710 llvm_unreachable("32-bit frames with WinEH must use FramePtr or BasePtr");
3711 }
3712 return MBBI;
3713 }
3714
getInitialCFAOffset(const MachineFunction & MF) const3715 int X86FrameLowering::getInitialCFAOffset(const MachineFunction &MF) const {
3716 return TRI->getSlotSize();
3717 }
3718
3719 Register
getInitialCFARegister(const MachineFunction & MF) const3720 X86FrameLowering::getInitialCFARegister(const MachineFunction &MF) const {
3721 return TRI->getDwarfRegNum(StackPtr, true);
3722 }
3723
3724 namespace {
3725 // Struct used by orderFrameObjects to help sort the stack objects.
3726 struct X86FrameSortingObject {
3727 bool IsValid = false; // true if we care about this Object.
3728 unsigned ObjectIndex = 0; // Index of Object into MFI list.
3729 unsigned ObjectSize = 0; // Size of Object in bytes.
3730 Align ObjectAlignment = Align(1); // Alignment of Object in bytes.
3731 unsigned ObjectNumUses = 0; // Object static number of uses.
3732 };
3733
3734 // The comparison function we use for std::sort to order our local
3735 // stack symbols. The current algorithm is to use an estimated
3736 // "density". This takes into consideration the size and number of
3737 // uses each object has in order to roughly minimize code size.
3738 // So, for example, an object of size 16B that is referenced 5 times
3739 // will get higher priority than 4 4B objects referenced 1 time each.
3740 // It's not perfect and we may be able to squeeze a few more bytes out of
3741 // it (for example : 0(esp) requires fewer bytes, symbols allocated at the
3742 // fringe end can have special consideration, given their size is less
3743 // important, etc.), but the algorithmic complexity grows too much to be
3744 // worth the extra gains we get. This gets us pretty close.
3745 // The final order leaves us with objects with highest priority going
3746 // at the end of our list.
3747 struct X86FrameSortingComparator {
operator ()__anonffc28aca0411::X86FrameSortingComparator3748 inline bool operator()(const X86FrameSortingObject &A,
3749 const X86FrameSortingObject &B) const {
3750 uint64_t DensityAScaled, DensityBScaled;
3751
3752 // For consistency in our comparison, all invalid objects are placed
3753 // at the end. This also allows us to stop walking when we hit the
3754 // first invalid item after it's all sorted.
3755 if (!A.IsValid)
3756 return false;
3757 if (!B.IsValid)
3758 return true;
3759
3760 // The density is calculated by doing :
3761 // (double)DensityA = A.ObjectNumUses / A.ObjectSize
3762 // (double)DensityB = B.ObjectNumUses / B.ObjectSize
3763 // Since this approach may cause inconsistencies in
3764 // the floating point <, >, == comparisons, depending on the floating
3765 // point model with which the compiler was built, we're going
3766 // to scale both sides by multiplying with
3767 // A.ObjectSize * B.ObjectSize. This ends up factoring away
3768 // the division and, with it, the need for any floating point
3769 // arithmetic.
3770 DensityAScaled = static_cast<uint64_t>(A.ObjectNumUses) *
3771 static_cast<uint64_t>(B.ObjectSize);
3772 DensityBScaled = static_cast<uint64_t>(B.ObjectNumUses) *
3773 static_cast<uint64_t>(A.ObjectSize);
3774
3775 // If the two densities are equal, prioritize highest alignment
3776 // objects. This allows for similar alignment objects
3777 // to be packed together (given the same density).
3778 // There's room for improvement here, also, since we can pack
3779 // similar alignment (different density) objects next to each
3780 // other to save padding. This will also require further
3781 // complexity/iterations, and the overall gain isn't worth it,
3782 // in general. Something to keep in mind, though.
3783 if (DensityAScaled == DensityBScaled)
3784 return A.ObjectAlignment < B.ObjectAlignment;
3785
3786 return DensityAScaled < DensityBScaled;
3787 }
3788 };
3789 } // namespace
3790
3791 // Order the symbols in the local stack.
3792 // We want to place the local stack objects in some sort of sensible order.
3793 // The heuristic we use is to try and pack them according to static number
3794 // of uses and size of object in order to minimize code size.
orderFrameObjects(const MachineFunction & MF,SmallVectorImpl<int> & ObjectsToAllocate) const3795 void X86FrameLowering::orderFrameObjects(
3796 const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
3797 const MachineFrameInfo &MFI = MF.getFrameInfo();
3798
3799 // Don't waste time if there's nothing to do.
3800 if (ObjectsToAllocate.empty())
3801 return;
3802
3803 // Create an array of all MFI objects. We won't need all of these
3804 // objects, but we're going to create a full array of them to make
3805 // it easier to index into when we're counting "uses" down below.
3806 // We want to be able to easily/cheaply access an object by simply
3807 // indexing into it, instead of having to search for it every time.
3808 std::vector<X86FrameSortingObject> SortingObjects(MFI.getObjectIndexEnd());
3809
3810 // Walk the objects we care about and mark them as such in our working
3811 // struct.
3812 for (auto &Obj : ObjectsToAllocate) {
3813 SortingObjects[Obj].IsValid = true;
3814 SortingObjects[Obj].ObjectIndex = Obj;
3815 SortingObjects[Obj].ObjectAlignment = MFI.getObjectAlign(Obj);
3816 // Set the size.
3817 int ObjectSize = MFI.getObjectSize(Obj);
3818 if (ObjectSize == 0)
3819 // Variable size. Just use 4.
3820 SortingObjects[Obj].ObjectSize = 4;
3821 else
3822 SortingObjects[Obj].ObjectSize = ObjectSize;
3823 }
3824
3825 // Count the number of uses for each object.
3826 for (auto &MBB : MF) {
3827 for (auto &MI : MBB) {
3828 if (MI.isDebugInstr())
3829 continue;
3830 for (const MachineOperand &MO : MI.operands()) {
3831 // Check to see if it's a local stack symbol.
3832 if (!MO.isFI())
3833 continue;
3834 int Index = MO.getIndex();
3835 // Check to see if it falls within our range, and is tagged
3836 // to require ordering.
3837 if (Index >= 0 && Index < MFI.getObjectIndexEnd() &&
3838 SortingObjects[Index].IsValid)
3839 SortingObjects[Index].ObjectNumUses++;
3840 }
3841 }
3842 }
3843
3844 // Sort the objects using X86FrameSortingAlgorithm (see its comment for
3845 // info).
3846 llvm::stable_sort(SortingObjects, X86FrameSortingComparator());
3847
3848 // Now modify the original list to represent the final order that
3849 // we want. The order will depend on whether we're going to access them
3850 // from the stack pointer or the frame pointer. For SP, the list should
3851 // end up with the END containing objects that we want with smaller offsets.
3852 // For FP, it should be flipped.
3853 int i = 0;
3854 for (auto &Obj : SortingObjects) {
3855 // All invalid items are sorted at the end, so it's safe to stop.
3856 if (!Obj.IsValid)
3857 break;
3858 ObjectsToAllocate[i++] = Obj.ObjectIndex;
3859 }
3860
3861 // Flip it if we're accessing off of the FP.
3862 if (!TRI->hasStackRealignment(MF) && hasFP(MF))
3863 std::reverse(ObjectsToAllocate.begin(), ObjectsToAllocate.end());
3864 }
3865
3866
getWinEHParentFrameOffset(const MachineFunction & MF) const3867 unsigned X86FrameLowering::getWinEHParentFrameOffset(const MachineFunction &MF) const {
3868 // RDX, the parent frame pointer, is homed into 16(%rsp) in the prologue.
3869 unsigned Offset = 16;
3870 // RBP is immediately pushed.
3871 Offset += SlotSize;
3872 // All callee-saved registers are then pushed.
3873 Offset += MF.getInfo<X86MachineFunctionInfo>()->getCalleeSavedFrameSize();
3874 // Every funclet allocates enough stack space for the largest outgoing call.
3875 Offset += getWinEHFuncletFrameSize(MF);
3876 return Offset;
3877 }
3878
processFunctionBeforeFrameFinalized(MachineFunction & MF,RegScavenger * RS) const3879 void X86FrameLowering::processFunctionBeforeFrameFinalized(
3880 MachineFunction &MF, RegScavenger *RS) const {
3881 // Mark the function as not having WinCFI. We will set it back to true in
3882 // emitPrologue if it gets called and emits CFI.
3883 MF.setHasWinCFI(false);
3884
3885 // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
3886 // aligned. The format doesn't support misaligned stack adjustments.
3887 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
3888 MF.getFrameInfo().ensureMaxAlignment(Align(SlotSize));
3889
3890 // If this function isn't doing Win64-style C++ EH, we don't need to do
3891 // anything.
3892 if (STI.is64Bit() && MF.hasEHFunclets() &&
3893 classifyEHPersonality(MF.getFunction().getPersonalityFn()) ==
3894 EHPersonality::MSVC_CXX) {
3895 adjustFrameForMsvcCxxEh(MF);
3896 }
3897 }
3898
adjustFrameForMsvcCxxEh(MachineFunction & MF) const3899 void X86FrameLowering::adjustFrameForMsvcCxxEh(MachineFunction &MF) const {
3900 // Win64 C++ EH needs to allocate the UnwindHelp object at some fixed offset
3901 // relative to RSP after the prologue. Find the offset of the last fixed
3902 // object, so that we can allocate a slot immediately following it. If there
3903 // were no fixed objects, use offset -SlotSize, which is immediately after the
3904 // return address. Fixed objects have negative frame indices.
3905 MachineFrameInfo &MFI = MF.getFrameInfo();
3906 WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
3907 int64_t MinFixedObjOffset = -SlotSize;
3908 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I)
3909 MinFixedObjOffset = std::min(MinFixedObjOffset, MFI.getObjectOffset(I));
3910
3911 for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
3912 for (WinEHHandlerType &H : TBME.HandlerArray) {
3913 int FrameIndex = H.CatchObj.FrameIndex;
3914 if (FrameIndex != INT_MAX) {
3915 // Ensure alignment.
3916 unsigned Align = MFI.getObjectAlign(FrameIndex).value();
3917 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % Align;
3918 MinFixedObjOffset -= MFI.getObjectSize(FrameIndex);
3919 MFI.setObjectOffset(FrameIndex, MinFixedObjOffset);
3920 }
3921 }
3922 }
3923
3924 // Ensure alignment.
3925 MinFixedObjOffset -= std::abs(MinFixedObjOffset) % 8;
3926 int64_t UnwindHelpOffset = MinFixedObjOffset - SlotSize;
3927 int UnwindHelpFI =
3928 MFI.CreateFixedObject(SlotSize, UnwindHelpOffset, /*IsImmutable=*/false);
3929 EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
3930
3931 // Store -2 into UnwindHelp on function entry. We have to scan forwards past
3932 // other frame setup instructions.
3933 MachineBasicBlock &MBB = MF.front();
3934 auto MBBI = MBB.begin();
3935 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
3936 ++MBBI;
3937
3938 DebugLoc DL = MBB.findDebugLoc(MBBI);
3939 addFrameReference(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mi32)),
3940 UnwindHelpFI)
3941 .addImm(-2);
3942 }
3943
getReturnProtector() const3944 const ReturnProtectorLowering *X86FrameLowering::getReturnProtector() const {
3945 return &RPL;
3946 }
3947
processFunctionBeforeFrameIndicesReplaced(MachineFunction & MF,RegScavenger * RS) const3948 void X86FrameLowering::processFunctionBeforeFrameIndicesReplaced(
3949 MachineFunction &MF, RegScavenger *RS) const {
3950 if (STI.is32Bit() && MF.hasEHFunclets())
3951 restoreWinEHStackPointersInParent(MF);
3952 }
3953
restoreWinEHStackPointersInParent(MachineFunction & MF) const3954 void X86FrameLowering::restoreWinEHStackPointersInParent(
3955 MachineFunction &MF) const {
3956 // 32-bit functions have to restore stack pointers when control is transferred
3957 // back to the parent function. These blocks are identified as eh pads that
3958 // are not funclet entries.
3959 bool IsSEH = isAsynchronousEHPersonality(
3960 classifyEHPersonality(MF.getFunction().getPersonalityFn()));
3961 for (MachineBasicBlock &MBB : MF) {
3962 bool NeedsRestore = MBB.isEHPad() && !MBB.isEHFuncletEntry();
3963 if (NeedsRestore)
3964 restoreWin32EHStackPointers(MBB, MBB.begin(), DebugLoc(),
3965 /*RestoreSP=*/IsSEH);
3966 }
3967 }
3968