10b57cec5SDimitry Andric //===-- X86FloatingPoint.cpp - Floating point Reg -> Stack converter ------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file defines the pass which converts floating point instructions from 100b57cec5SDimitry Andric // pseudo registers into register stack instructions. This pass uses live 110b57cec5SDimitry Andric // variable information to indicate where the FPn registers are used and their 120b57cec5SDimitry Andric // lifetimes. 130b57cec5SDimitry Andric // 140b57cec5SDimitry Andric // The x87 hardware tracks liveness of the stack registers, so it is necessary 150b57cec5SDimitry Andric // to implement exact liveness tracking between basic blocks. The CFG edges are 160b57cec5SDimitry Andric // partitioned into bundles where the same FP registers must be live in 170b57cec5SDimitry Andric // identical stack positions. Instructions are inserted at the end of each basic 180b57cec5SDimitry Andric // block to rearrange the live registers to match the outgoing bundle. 190b57cec5SDimitry Andric // 200b57cec5SDimitry Andric // This approach avoids splitting critical edges at the potential cost of more 210b57cec5SDimitry Andric // live register shuffling instructions when critical edges are present. 220b57cec5SDimitry Andric // 230b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 240b57cec5SDimitry Andric 250b57cec5SDimitry Andric #include "X86.h" 260b57cec5SDimitry Andric #include "X86InstrInfo.h" 270b57cec5SDimitry Andric #include "llvm/ADT/DepthFirstIterator.h" 280b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 290b57cec5SDimitry Andric #include "llvm/ADT/SmallSet.h" 300b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h" 310b57cec5SDimitry Andric #include "llvm/ADT/Statistic.h" 320b57cec5SDimitry Andric #include "llvm/CodeGen/EdgeBundles.h" 33*0fca6ea1SDimitry Andric #include "llvm/CodeGen/LiveRegUnits.h" 340b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunctionPass.h" 350b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstrBuilder.h" 360b57cec5SDimitry Andric #include "llvm/CodeGen/MachineRegisterInfo.h" 370b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h" 380b57cec5SDimitry Andric #include "llvm/CodeGen/TargetInstrInfo.h" 390b57cec5SDimitry Andric #include "llvm/CodeGen/TargetSubtargetInfo.h" 400b57cec5SDimitry Andric #include "llvm/Config/llvm-config.h" 410b57cec5SDimitry Andric #include "llvm/IR/InlineAsm.h" 42480093f4SDimitry Andric #include "llvm/InitializePasses.h" 430b57cec5SDimitry Andric #include "llvm/Support/Debug.h" 440b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 450b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h" 460b57cec5SDimitry Andric #include "llvm/Target/TargetMachine.h" 470b57cec5SDimitry Andric #include <algorithm> 480b57cec5SDimitry Andric #include <bitset> 490b57cec5SDimitry Andric using namespace llvm; 500b57cec5SDimitry Andric 510b57cec5SDimitry Andric #define DEBUG_TYPE "x86-codegen" 520b57cec5SDimitry Andric 530b57cec5SDimitry Andric STATISTIC(NumFXCH, "Number of fxch instructions inserted"); 540b57cec5SDimitry Andric STATISTIC(NumFP , "Number of floating point instructions"); 550b57cec5SDimitry Andric 560b57cec5SDimitry Andric namespace { 570b57cec5SDimitry Andric const unsigned ScratchFPReg = 7; 580b57cec5SDimitry Andric 590b57cec5SDimitry Andric struct FPS : public MachineFunctionPass { 600b57cec5SDimitry Andric static char ID; 610b57cec5SDimitry Andric FPS() : MachineFunctionPass(ID) { 620b57cec5SDimitry Andric // This is really only to keep valgrind quiet. 630b57cec5SDimitry Andric // The logic in isLive() is too much for it. 640b57cec5SDimitry Andric memset(Stack, 0, sizeof(Stack)); 650b57cec5SDimitry Andric memset(RegMap, 0, sizeof(RegMap)); 660b57cec5SDimitry Andric } 670b57cec5SDimitry Andric 680b57cec5SDimitry Andric void getAnalysisUsage(AnalysisUsage &AU) const override { 690b57cec5SDimitry Andric AU.setPreservesCFG(); 700b57cec5SDimitry Andric AU.addRequired<EdgeBundles>(); 710b57cec5SDimitry Andric AU.addPreservedID(MachineLoopInfoID); 720b57cec5SDimitry Andric AU.addPreservedID(MachineDominatorsID); 730b57cec5SDimitry Andric MachineFunctionPass::getAnalysisUsage(AU); 740b57cec5SDimitry Andric } 750b57cec5SDimitry Andric 760b57cec5SDimitry Andric bool runOnMachineFunction(MachineFunction &MF) override; 770b57cec5SDimitry Andric 780b57cec5SDimitry Andric MachineFunctionProperties getRequiredProperties() const override { 790b57cec5SDimitry Andric return MachineFunctionProperties().set( 800b57cec5SDimitry Andric MachineFunctionProperties::Property::NoVRegs); 810b57cec5SDimitry Andric } 820b57cec5SDimitry Andric 830b57cec5SDimitry Andric StringRef getPassName() const override { return "X86 FP Stackifier"; } 840b57cec5SDimitry Andric 850b57cec5SDimitry Andric private: 86480093f4SDimitry Andric const TargetInstrInfo *TII = nullptr; // Machine instruction info. 870b57cec5SDimitry Andric 880b57cec5SDimitry Andric // Two CFG edges are related if they leave the same block, or enter the same 890b57cec5SDimitry Andric // block. The transitive closure of an edge under this relation is a 900b57cec5SDimitry Andric // LiveBundle. It represents a set of CFG edges where the live FP stack 910b57cec5SDimitry Andric // registers must be allocated identically in the x87 stack. 920b57cec5SDimitry Andric // 930b57cec5SDimitry Andric // A LiveBundle is usually all the edges leaving a block, or all the edges 940b57cec5SDimitry Andric // entering a block, but it can contain more edges if critical edges are 950b57cec5SDimitry Andric // present. 960b57cec5SDimitry Andric // 970b57cec5SDimitry Andric // The set of live FP registers in a LiveBundle is calculated by bundleCFG, 980b57cec5SDimitry Andric // but the exact mapping of FP registers to stack slots is fixed later. 990b57cec5SDimitry Andric struct LiveBundle { 1000b57cec5SDimitry Andric // Bit mask of live FP registers. Bit 0 = FP0, bit 1 = FP1, &c. 10181ad6265SDimitry Andric unsigned Mask = 0; 1020b57cec5SDimitry Andric 1030b57cec5SDimitry Andric // Number of pre-assigned live registers in FixStack. This is 0 when the 1040b57cec5SDimitry Andric // stack order has not yet been fixed. 10581ad6265SDimitry Andric unsigned FixCount = 0; 1060b57cec5SDimitry Andric 1070b57cec5SDimitry Andric // Assigned stack order for live-in registers. 1080b57cec5SDimitry Andric // FixStack[i] == getStackEntry(i) for all i < FixCount. 1090b57cec5SDimitry Andric unsigned char FixStack[8]; 1100b57cec5SDimitry Andric 11181ad6265SDimitry Andric LiveBundle() = default; 1120b57cec5SDimitry Andric 1130b57cec5SDimitry Andric // Have the live registers been assigned a stack order yet? 1140b57cec5SDimitry Andric bool isFixed() const { return !Mask || FixCount; } 1150b57cec5SDimitry Andric }; 1160b57cec5SDimitry Andric 1170b57cec5SDimitry Andric // Numbered LiveBundle structs. LiveBundles[0] is used for all CFG edges 1180b57cec5SDimitry Andric // with no live FP registers. 1190b57cec5SDimitry Andric SmallVector<LiveBundle, 8> LiveBundles; 1200b57cec5SDimitry Andric 1210b57cec5SDimitry Andric // The edge bundle analysis provides indices into the LiveBundles vector. 122480093f4SDimitry Andric EdgeBundles *Bundles = nullptr; 1230b57cec5SDimitry Andric 1240b57cec5SDimitry Andric // Return a bitmask of FP registers in block's live-in list. 1250b57cec5SDimitry Andric static unsigned calcLiveInMask(MachineBasicBlock *MBB, bool RemoveFPs) { 1260b57cec5SDimitry Andric unsigned Mask = 0; 1270b57cec5SDimitry Andric for (MachineBasicBlock::livein_iterator I = MBB->livein_begin(); 1280b57cec5SDimitry Andric I != MBB->livein_end(); ) { 1290b57cec5SDimitry Andric MCPhysReg Reg = I->PhysReg; 1300b57cec5SDimitry Andric static_assert(X86::FP6 - X86::FP0 == 6, "sequential regnums"); 1310b57cec5SDimitry Andric if (Reg >= X86::FP0 && Reg <= X86::FP6) { 1320b57cec5SDimitry Andric Mask |= 1 << (Reg - X86::FP0); 1330b57cec5SDimitry Andric if (RemoveFPs) { 1340b57cec5SDimitry Andric I = MBB->removeLiveIn(I); 1350b57cec5SDimitry Andric continue; 1360b57cec5SDimitry Andric } 1370b57cec5SDimitry Andric } 1380b57cec5SDimitry Andric ++I; 1390b57cec5SDimitry Andric } 1400b57cec5SDimitry Andric return Mask; 1410b57cec5SDimitry Andric } 1420b57cec5SDimitry Andric 1430b57cec5SDimitry Andric // Partition all the CFG edges into LiveBundles. 1440b57cec5SDimitry Andric void bundleCFGRecomputeKillFlags(MachineFunction &MF); 1450b57cec5SDimitry Andric 146480093f4SDimitry Andric MachineBasicBlock *MBB = nullptr; // Current basic block 1470b57cec5SDimitry Andric 1480b57cec5SDimitry Andric // The hardware keeps track of how many FP registers are live, so we have 1490b57cec5SDimitry Andric // to model that exactly. Usually, each live register corresponds to an 1500b57cec5SDimitry Andric // FP<n> register, but when dealing with calls, returns, and inline 1510b57cec5SDimitry Andric // assembly, it is sometimes necessary to have live scratch registers. 1520b57cec5SDimitry Andric unsigned Stack[8]; // FP<n> Registers in each stack slot... 153480093f4SDimitry Andric unsigned StackTop = 0; // The current top of the FP stack. 1540b57cec5SDimitry Andric 1550b57cec5SDimitry Andric enum { 1560b57cec5SDimitry Andric NumFPRegs = 8 // Including scratch pseudo-registers. 1570b57cec5SDimitry Andric }; 1580b57cec5SDimitry Andric 1590b57cec5SDimitry Andric // For each live FP<n> register, point to its Stack[] entry. 1600b57cec5SDimitry Andric // The first entries correspond to FP0-FP6, the rest are scratch registers 1610b57cec5SDimitry Andric // used when we need slightly different live registers than what the 1620b57cec5SDimitry Andric // register allocator thinks. 1630b57cec5SDimitry Andric unsigned RegMap[NumFPRegs]; 1640b57cec5SDimitry Andric 1650b57cec5SDimitry Andric // Set up our stack model to match the incoming registers to MBB. 1660b57cec5SDimitry Andric void setupBlockStack(); 1670b57cec5SDimitry Andric 1680b57cec5SDimitry Andric // Shuffle live registers to match the expectations of successor blocks. 1690b57cec5SDimitry Andric void finishBlockStack(); 1700b57cec5SDimitry Andric 1710b57cec5SDimitry Andric #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1720b57cec5SDimitry Andric void dumpStack() const { 1730b57cec5SDimitry Andric dbgs() << "Stack contents:"; 1740b57cec5SDimitry Andric for (unsigned i = 0; i != StackTop; ++i) { 1750b57cec5SDimitry Andric dbgs() << " FP" << Stack[i]; 1760b57cec5SDimitry Andric assert(RegMap[Stack[i]] == i && "Stack[] doesn't match RegMap[]!"); 1770b57cec5SDimitry Andric } 1780b57cec5SDimitry Andric } 1790b57cec5SDimitry Andric #endif 1800b57cec5SDimitry Andric 1810b57cec5SDimitry Andric /// getSlot - Return the stack slot number a particular register number is 1820b57cec5SDimitry Andric /// in. 1830b57cec5SDimitry Andric unsigned getSlot(unsigned RegNo) const { 1840b57cec5SDimitry Andric assert(RegNo < NumFPRegs && "Regno out of range!"); 1850b57cec5SDimitry Andric return RegMap[RegNo]; 1860b57cec5SDimitry Andric } 1870b57cec5SDimitry Andric 1880b57cec5SDimitry Andric /// isLive - Is RegNo currently live in the stack? 1890b57cec5SDimitry Andric bool isLive(unsigned RegNo) const { 1900b57cec5SDimitry Andric unsigned Slot = getSlot(RegNo); 1910b57cec5SDimitry Andric return Slot < StackTop && Stack[Slot] == RegNo; 1920b57cec5SDimitry Andric } 1930b57cec5SDimitry Andric 1940b57cec5SDimitry Andric /// getStackEntry - Return the X86::FP<n> register in register ST(i). 1950b57cec5SDimitry Andric unsigned getStackEntry(unsigned STi) const { 1960b57cec5SDimitry Andric if (STi >= StackTop) 1970b57cec5SDimitry Andric report_fatal_error("Access past stack top!"); 1980b57cec5SDimitry Andric return Stack[StackTop-1-STi]; 1990b57cec5SDimitry Andric } 2000b57cec5SDimitry Andric 2010b57cec5SDimitry Andric /// getSTReg - Return the X86::ST(i) register which contains the specified 2020b57cec5SDimitry Andric /// FP<RegNo> register. 2030b57cec5SDimitry Andric unsigned getSTReg(unsigned RegNo) const { 2040b57cec5SDimitry Andric return StackTop - 1 - getSlot(RegNo) + X86::ST0; 2050b57cec5SDimitry Andric } 2060b57cec5SDimitry Andric 2070b57cec5SDimitry Andric // pushReg - Push the specified FP<n> register onto the stack. 2080b57cec5SDimitry Andric void pushReg(unsigned Reg) { 2090b57cec5SDimitry Andric assert(Reg < NumFPRegs && "Register number out of range!"); 2100b57cec5SDimitry Andric if (StackTop >= 8) 2110b57cec5SDimitry Andric report_fatal_error("Stack overflow!"); 2120b57cec5SDimitry Andric Stack[StackTop] = Reg; 2130b57cec5SDimitry Andric RegMap[Reg] = StackTop++; 2140b57cec5SDimitry Andric } 2150b57cec5SDimitry Andric 2160b57cec5SDimitry Andric // popReg - Pop a register from the stack. 2170b57cec5SDimitry Andric void popReg() { 2180b57cec5SDimitry Andric if (StackTop == 0) 2190b57cec5SDimitry Andric report_fatal_error("Cannot pop empty stack!"); 2200b57cec5SDimitry Andric RegMap[Stack[--StackTop]] = ~0; // Update state 2210b57cec5SDimitry Andric } 2220b57cec5SDimitry Andric 2230b57cec5SDimitry Andric bool isAtTop(unsigned RegNo) const { return getSlot(RegNo) == StackTop-1; } 2240b57cec5SDimitry Andric void moveToTop(unsigned RegNo, MachineBasicBlock::iterator I) { 2250b57cec5SDimitry Andric DebugLoc dl = I == MBB->end() ? DebugLoc() : I->getDebugLoc(); 2260b57cec5SDimitry Andric if (isAtTop(RegNo)) return; 2270b57cec5SDimitry Andric 2280b57cec5SDimitry Andric unsigned STReg = getSTReg(RegNo); 2290b57cec5SDimitry Andric unsigned RegOnTop = getStackEntry(0); 2300b57cec5SDimitry Andric 2310b57cec5SDimitry Andric // Swap the slots the regs are in. 2320b57cec5SDimitry Andric std::swap(RegMap[RegNo], RegMap[RegOnTop]); 2330b57cec5SDimitry Andric 2340b57cec5SDimitry Andric // Swap stack slot contents. 2350b57cec5SDimitry Andric if (RegMap[RegOnTop] >= StackTop) 2360b57cec5SDimitry Andric report_fatal_error("Access past stack top!"); 2370b57cec5SDimitry Andric std::swap(Stack[RegMap[RegOnTop]], Stack[StackTop-1]); 2380b57cec5SDimitry Andric 2390b57cec5SDimitry Andric // Emit an fxch to update the runtime processors version of the state. 2400b57cec5SDimitry Andric BuildMI(*MBB, I, dl, TII->get(X86::XCH_F)).addReg(STReg); 2410b57cec5SDimitry Andric ++NumFXCH; 2420b57cec5SDimitry Andric } 2430b57cec5SDimitry Andric 2440b57cec5SDimitry Andric void duplicateToTop(unsigned RegNo, unsigned AsReg, 2450b57cec5SDimitry Andric MachineBasicBlock::iterator I) { 2460b57cec5SDimitry Andric DebugLoc dl = I == MBB->end() ? DebugLoc() : I->getDebugLoc(); 2470b57cec5SDimitry Andric unsigned STReg = getSTReg(RegNo); 2480b57cec5SDimitry Andric pushReg(AsReg); // New register on top of stack 2490b57cec5SDimitry Andric 2500b57cec5SDimitry Andric BuildMI(*MBB, I, dl, TII->get(X86::LD_Frr)).addReg(STReg); 2510b57cec5SDimitry Andric } 2520b57cec5SDimitry Andric 2530b57cec5SDimitry Andric /// popStackAfter - Pop the current value off of the top of the FP stack 2540b57cec5SDimitry Andric /// after the specified instruction. 2550b57cec5SDimitry Andric void popStackAfter(MachineBasicBlock::iterator &I); 2560b57cec5SDimitry Andric 2570b57cec5SDimitry Andric /// freeStackSlotAfter - Free the specified register from the register 2580b57cec5SDimitry Andric /// stack, so that it is no longer in a register. If the register is 2590b57cec5SDimitry Andric /// currently at the top of the stack, we just pop the current instruction, 2600b57cec5SDimitry Andric /// otherwise we store the current top-of-stack into the specified slot, 2610b57cec5SDimitry Andric /// then pop the top of stack. 2620b57cec5SDimitry Andric void freeStackSlotAfter(MachineBasicBlock::iterator &I, unsigned Reg); 2630b57cec5SDimitry Andric 2640b57cec5SDimitry Andric /// freeStackSlotBefore - Just the pop, no folding. Return the inserted 2650b57cec5SDimitry Andric /// instruction. 2660b57cec5SDimitry Andric MachineBasicBlock::iterator 2670b57cec5SDimitry Andric freeStackSlotBefore(MachineBasicBlock::iterator I, unsigned FPRegNo); 2680b57cec5SDimitry Andric 2690b57cec5SDimitry Andric /// Adjust the live registers to be the set in Mask. 2700b57cec5SDimitry Andric void adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I); 2710b57cec5SDimitry Andric 2720b57cec5SDimitry Andric /// Shuffle the top FixCount stack entries such that FP reg FixStack[0] is 2730b57cec5SDimitry Andric /// st(0), FP reg FixStack[1] is st(1) etc. 2740b57cec5SDimitry Andric void shuffleStackTop(const unsigned char *FixStack, unsigned FixCount, 2750b57cec5SDimitry Andric MachineBasicBlock::iterator I); 2760b57cec5SDimitry Andric 2770b57cec5SDimitry Andric bool processBasicBlock(MachineFunction &MF, MachineBasicBlock &MBB); 2780b57cec5SDimitry Andric 2790b57cec5SDimitry Andric void handleCall(MachineBasicBlock::iterator &I); 2800b57cec5SDimitry Andric void handleReturn(MachineBasicBlock::iterator &I); 2810b57cec5SDimitry Andric void handleZeroArgFP(MachineBasicBlock::iterator &I); 2820b57cec5SDimitry Andric void handleOneArgFP(MachineBasicBlock::iterator &I); 2830b57cec5SDimitry Andric void handleOneArgFPRW(MachineBasicBlock::iterator &I); 2840b57cec5SDimitry Andric void handleTwoArgFP(MachineBasicBlock::iterator &I); 2850b57cec5SDimitry Andric void handleCompareFP(MachineBasicBlock::iterator &I); 2860b57cec5SDimitry Andric void handleCondMovFP(MachineBasicBlock::iterator &I); 2870b57cec5SDimitry Andric void handleSpecialFP(MachineBasicBlock::iterator &I); 2880b57cec5SDimitry Andric 2890b57cec5SDimitry Andric // Check if a COPY instruction is using FP registers. 2900b57cec5SDimitry Andric static bool isFPCopy(MachineInstr &MI) { 2918bcb0991SDimitry Andric Register DstReg = MI.getOperand(0).getReg(); 2928bcb0991SDimitry Andric Register SrcReg = MI.getOperand(1).getReg(); 2930b57cec5SDimitry Andric 2940b57cec5SDimitry Andric return X86::RFP80RegClass.contains(DstReg) || 2950b57cec5SDimitry Andric X86::RFP80RegClass.contains(SrcReg); 2960b57cec5SDimitry Andric } 2970b57cec5SDimitry Andric 2980b57cec5SDimitry Andric void setKillFlags(MachineBasicBlock &MBB) const; 2990b57cec5SDimitry Andric }; 3000b57cec5SDimitry Andric } 3010b57cec5SDimitry Andric 3020b57cec5SDimitry Andric char FPS::ID = 0; 3030b57cec5SDimitry Andric 3040b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(FPS, DEBUG_TYPE, "X86 FP Stackifier", 3050b57cec5SDimitry Andric false, false) 3060b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(EdgeBundles) 3070b57cec5SDimitry Andric INITIALIZE_PASS_END(FPS, DEBUG_TYPE, "X86 FP Stackifier", 3080b57cec5SDimitry Andric false, false) 3090b57cec5SDimitry Andric 3100b57cec5SDimitry Andric FunctionPass *llvm::createX86FloatingPointStackifierPass() { return new FPS(); } 3110b57cec5SDimitry Andric 3120b57cec5SDimitry Andric /// getFPReg - Return the X86::FPx register number for the specified operand. 3130b57cec5SDimitry Andric /// For example, this returns 3 for X86::FP3. 3140b57cec5SDimitry Andric static unsigned getFPReg(const MachineOperand &MO) { 3150b57cec5SDimitry Andric assert(MO.isReg() && "Expected an FP register!"); 3168bcb0991SDimitry Andric Register Reg = MO.getReg(); 3170b57cec5SDimitry Andric assert(Reg >= X86::FP0 && Reg <= X86::FP6 && "Expected FP register!"); 3180b57cec5SDimitry Andric return Reg - X86::FP0; 3190b57cec5SDimitry Andric } 3200b57cec5SDimitry Andric 3210b57cec5SDimitry Andric /// runOnMachineFunction - Loop over all of the basic blocks, transforming FP 3220b57cec5SDimitry Andric /// register references into FP stack references. 3230b57cec5SDimitry Andric /// 3240b57cec5SDimitry Andric bool FPS::runOnMachineFunction(MachineFunction &MF) { 3250b57cec5SDimitry Andric // We only need to run this pass if there are any FP registers used in this 3260b57cec5SDimitry Andric // function. If it is all integer, there is nothing for us to do! 3270b57cec5SDimitry Andric bool FPIsUsed = false; 3280b57cec5SDimitry Andric 3290b57cec5SDimitry Andric static_assert(X86::FP6 == X86::FP0+6, "Register enums aren't sorted right!"); 3300b57cec5SDimitry Andric const MachineRegisterInfo &MRI = MF.getRegInfo(); 3310b57cec5SDimitry Andric for (unsigned i = 0; i <= 6; ++i) 3320b57cec5SDimitry Andric if (!MRI.reg_nodbg_empty(X86::FP0 + i)) { 3330b57cec5SDimitry Andric FPIsUsed = true; 3340b57cec5SDimitry Andric break; 3350b57cec5SDimitry Andric } 3360b57cec5SDimitry Andric 3370b57cec5SDimitry Andric // Early exit. 3380b57cec5SDimitry Andric if (!FPIsUsed) return false; 3390b57cec5SDimitry Andric 3400b57cec5SDimitry Andric Bundles = &getAnalysis<EdgeBundles>(); 3410b57cec5SDimitry Andric TII = MF.getSubtarget().getInstrInfo(); 3420b57cec5SDimitry Andric 3430b57cec5SDimitry Andric // Prepare cross-MBB liveness. 3440b57cec5SDimitry Andric bundleCFGRecomputeKillFlags(MF); 3450b57cec5SDimitry Andric 3460b57cec5SDimitry Andric StackTop = 0; 3470b57cec5SDimitry Andric 3480b57cec5SDimitry Andric // Process the function in depth first order so that we process at least one 3490b57cec5SDimitry Andric // of the predecessors for every reachable block in the function. 3500b57cec5SDimitry Andric df_iterator_default_set<MachineBasicBlock*> Processed; 3510b57cec5SDimitry Andric MachineBasicBlock *Entry = &MF.front(); 3520b57cec5SDimitry Andric 3530b57cec5SDimitry Andric LiveBundle &Bundle = 3540b57cec5SDimitry Andric LiveBundles[Bundles->getBundle(Entry->getNumber(), false)]; 3550b57cec5SDimitry Andric 3560b57cec5SDimitry Andric // In regcall convention, some FP registers may not be passed through 3570b57cec5SDimitry Andric // the stack, so they will need to be assigned to the stack first 3580b57cec5SDimitry Andric if ((Entry->getParent()->getFunction().getCallingConv() == 3590b57cec5SDimitry Andric CallingConv::X86_RegCall) && (Bundle.Mask && !Bundle.FixCount)) { 3600b57cec5SDimitry Andric // In the register calling convention, up to one FP argument could be 3610b57cec5SDimitry Andric // saved in the first FP register. 3620b57cec5SDimitry Andric // If bundle.mask is non-zero and Bundle.FixCount is zero, it means 3630b57cec5SDimitry Andric // that the FP registers contain arguments. 3640b57cec5SDimitry Andric // The actual value is passed in FP0. 3650b57cec5SDimitry Andric // Here we fix the stack and mark FP0 as pre-assigned register. 3660b57cec5SDimitry Andric assert((Bundle.Mask & 0xFE) == 0 && 3670b57cec5SDimitry Andric "Only FP0 could be passed as an argument"); 3680b57cec5SDimitry Andric Bundle.FixCount = 1; 3690b57cec5SDimitry Andric Bundle.FixStack[0] = 0; 3700b57cec5SDimitry Andric } 3710b57cec5SDimitry Andric 3720b57cec5SDimitry Andric bool Changed = false; 3730b57cec5SDimitry Andric for (MachineBasicBlock *BB : depth_first_ext(Entry, Processed)) 3740b57cec5SDimitry Andric Changed |= processBasicBlock(MF, *BB); 3750b57cec5SDimitry Andric 3760b57cec5SDimitry Andric // Process any unreachable blocks in arbitrary order now. 3770b57cec5SDimitry Andric if (MF.size() != Processed.size()) 3780b57cec5SDimitry Andric for (MachineBasicBlock &BB : MF) 3790b57cec5SDimitry Andric if (Processed.insert(&BB).second) 3800b57cec5SDimitry Andric Changed |= processBasicBlock(MF, BB); 3810b57cec5SDimitry Andric 3820b57cec5SDimitry Andric LiveBundles.clear(); 3830b57cec5SDimitry Andric 3840b57cec5SDimitry Andric return Changed; 3850b57cec5SDimitry Andric } 3860b57cec5SDimitry Andric 3870b57cec5SDimitry Andric /// bundleCFG - Scan all the basic blocks to determine consistent live-in and 3880b57cec5SDimitry Andric /// live-out sets for the FP registers. Consistent means that the set of 3890b57cec5SDimitry Andric /// registers live-out from a block is identical to the live-in set of all 3900b57cec5SDimitry Andric /// successors. This is not enforced by the normal live-in lists since 3910b57cec5SDimitry Andric /// registers may be implicitly defined, or not used by all successors. 3920b57cec5SDimitry Andric void FPS::bundleCFGRecomputeKillFlags(MachineFunction &MF) { 3930b57cec5SDimitry Andric assert(LiveBundles.empty() && "Stale data in LiveBundles"); 3940b57cec5SDimitry Andric LiveBundles.resize(Bundles->getNumBundles()); 3950b57cec5SDimitry Andric 3960b57cec5SDimitry Andric // Gather the actual live-in masks for all MBBs. 3970b57cec5SDimitry Andric for (MachineBasicBlock &MBB : MF) { 3980b57cec5SDimitry Andric setKillFlags(MBB); 3990b57cec5SDimitry Andric 4000b57cec5SDimitry Andric const unsigned Mask = calcLiveInMask(&MBB, false); 4010b57cec5SDimitry Andric if (!Mask) 4020b57cec5SDimitry Andric continue; 4030b57cec5SDimitry Andric // Update MBB ingoing bundle mask. 4040b57cec5SDimitry Andric LiveBundles[Bundles->getBundle(MBB.getNumber(), false)].Mask |= Mask; 4050b57cec5SDimitry Andric } 4060b57cec5SDimitry Andric } 4070b57cec5SDimitry Andric 4080b57cec5SDimitry Andric /// processBasicBlock - Loop over all of the instructions in the basic block, 4090b57cec5SDimitry Andric /// transforming FP instructions into their stack form. 4100b57cec5SDimitry Andric /// 4110b57cec5SDimitry Andric bool FPS::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { 4120b57cec5SDimitry Andric bool Changed = false; 4130b57cec5SDimitry Andric MBB = &BB; 4140b57cec5SDimitry Andric 4150b57cec5SDimitry Andric setupBlockStack(); 4160b57cec5SDimitry Andric 4170b57cec5SDimitry Andric for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) { 4180b57cec5SDimitry Andric MachineInstr &MI = *I; 4190b57cec5SDimitry Andric uint64_t Flags = MI.getDesc().TSFlags; 4200b57cec5SDimitry Andric 4210b57cec5SDimitry Andric unsigned FPInstClass = Flags & X86II::FPTypeMask; 4220b57cec5SDimitry Andric if (MI.isInlineAsm()) 4230b57cec5SDimitry Andric FPInstClass = X86II::SpecialFP; 4240b57cec5SDimitry Andric 4250b57cec5SDimitry Andric if (MI.isCopy() && isFPCopy(MI)) 4260b57cec5SDimitry Andric FPInstClass = X86II::SpecialFP; 4270b57cec5SDimitry Andric 4280b57cec5SDimitry Andric if (MI.isImplicitDef() && 4290b57cec5SDimitry Andric X86::RFP80RegClass.contains(MI.getOperand(0).getReg())) 4300b57cec5SDimitry Andric FPInstClass = X86II::SpecialFP; 4310b57cec5SDimitry Andric 4320b57cec5SDimitry Andric if (MI.isCall()) 4330b57cec5SDimitry Andric FPInstClass = X86II::SpecialFP; 4340b57cec5SDimitry Andric 4350b57cec5SDimitry Andric if (FPInstClass == X86II::NotFP) 4360b57cec5SDimitry Andric continue; // Efficiently ignore non-fp insts! 4370b57cec5SDimitry Andric 4380b57cec5SDimitry Andric MachineInstr *PrevMI = nullptr; 4390b57cec5SDimitry Andric if (I != BB.begin()) 4400b57cec5SDimitry Andric PrevMI = &*std::prev(I); 4410b57cec5SDimitry Andric 4420b57cec5SDimitry Andric ++NumFP; // Keep track of # of pseudo instrs 4430b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "\nFPInst:\t" << MI); 4440b57cec5SDimitry Andric 4450b57cec5SDimitry Andric // Get dead variables list now because the MI pointer may be deleted as part 4460b57cec5SDimitry Andric // of processing! 4470b57cec5SDimitry Andric SmallVector<unsigned, 8> DeadRegs; 4484824e7fdSDimitry Andric for (const MachineOperand &MO : MI.operands()) 4490b57cec5SDimitry Andric if (MO.isReg() && MO.isDead()) 4500b57cec5SDimitry Andric DeadRegs.push_back(MO.getReg()); 4510b57cec5SDimitry Andric 4520b57cec5SDimitry Andric switch (FPInstClass) { 4530b57cec5SDimitry Andric case X86II::ZeroArgFP: handleZeroArgFP(I); break; 4540b57cec5SDimitry Andric case X86II::OneArgFP: handleOneArgFP(I); break; // fstp ST(0) 4550b57cec5SDimitry Andric case X86II::OneArgFPRW: handleOneArgFPRW(I); break; // ST(0) = fsqrt(ST(0)) 4560b57cec5SDimitry Andric case X86II::TwoArgFP: handleTwoArgFP(I); break; 4570b57cec5SDimitry Andric case X86II::CompareFP: handleCompareFP(I); break; 4580b57cec5SDimitry Andric case X86II::CondMovFP: handleCondMovFP(I); break; 4590b57cec5SDimitry Andric case X86II::SpecialFP: handleSpecialFP(I); break; 4600b57cec5SDimitry Andric default: llvm_unreachable("Unknown FP Type!"); 4610b57cec5SDimitry Andric } 4620b57cec5SDimitry Andric 4630b57cec5SDimitry Andric // Check to see if any of the values defined by this instruction are dead 4640b57cec5SDimitry Andric // after definition. If so, pop them. 465cb14a3feSDimitry Andric for (unsigned Reg : DeadRegs) { 4660b57cec5SDimitry Andric // Check if Reg is live on the stack. An inline-asm register operand that 4670b57cec5SDimitry Andric // is in the clobber list and marked dead might not be live on the stack. 4680b57cec5SDimitry Andric static_assert(X86::FP7 - X86::FP0 == 7, "sequential FP regnumbers"); 4690b57cec5SDimitry Andric if (Reg >= X86::FP0 && Reg <= X86::FP6 && isLive(Reg-X86::FP0)) { 4700b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Register FP#" << Reg - X86::FP0 << " is dead!\n"); 4710b57cec5SDimitry Andric freeStackSlotAfter(I, Reg-X86::FP0); 4720b57cec5SDimitry Andric } 4730b57cec5SDimitry Andric } 4740b57cec5SDimitry Andric 4750b57cec5SDimitry Andric // Print out all of the instructions expanded to if -debug 4760b57cec5SDimitry Andric LLVM_DEBUG({ 4770b57cec5SDimitry Andric MachineBasicBlock::iterator PrevI = PrevMI; 4780b57cec5SDimitry Andric if (I == PrevI) { 4790b57cec5SDimitry Andric dbgs() << "Just deleted pseudo instruction\n"; 4800b57cec5SDimitry Andric } else { 4810b57cec5SDimitry Andric MachineBasicBlock::iterator Start = I; 4820b57cec5SDimitry Andric // Rewind to first instruction newly inserted. 4830b57cec5SDimitry Andric while (Start != BB.begin() && std::prev(Start) != PrevI) 4840b57cec5SDimitry Andric --Start; 4850b57cec5SDimitry Andric dbgs() << "Inserted instructions:\n\t"; 4860b57cec5SDimitry Andric Start->print(dbgs()); 4870b57cec5SDimitry Andric while (++Start != std::next(I)) { 4880b57cec5SDimitry Andric } 4890b57cec5SDimitry Andric } 4900b57cec5SDimitry Andric dumpStack(); 4910b57cec5SDimitry Andric }); 4920b57cec5SDimitry Andric (void)PrevMI; 4930b57cec5SDimitry Andric 4940b57cec5SDimitry Andric Changed = true; 4950b57cec5SDimitry Andric } 4960b57cec5SDimitry Andric 4970b57cec5SDimitry Andric finishBlockStack(); 4980b57cec5SDimitry Andric 4990b57cec5SDimitry Andric return Changed; 5000b57cec5SDimitry Andric } 5010b57cec5SDimitry Andric 5020b57cec5SDimitry Andric /// setupBlockStack - Use the live bundles to set up our model of the stack 5030b57cec5SDimitry Andric /// to match predecessors' live out stack. 5040b57cec5SDimitry Andric void FPS::setupBlockStack() { 5050b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "\nSetting up live-ins for " << printMBBReference(*MBB) 5060b57cec5SDimitry Andric << " derived from " << MBB->getName() << ".\n"); 5070b57cec5SDimitry Andric StackTop = 0; 5080b57cec5SDimitry Andric // Get the live-in bundle for MBB. 5090b57cec5SDimitry Andric const LiveBundle &Bundle = 5100b57cec5SDimitry Andric LiveBundles[Bundles->getBundle(MBB->getNumber(), false)]; 5110b57cec5SDimitry Andric 5120b57cec5SDimitry Andric if (!Bundle.Mask) { 5130b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Block has no FP live-ins.\n"); 5140b57cec5SDimitry Andric return; 5150b57cec5SDimitry Andric } 5160b57cec5SDimitry Andric 5170b57cec5SDimitry Andric // Depth-first iteration should ensure that we always have an assigned stack. 5180b57cec5SDimitry Andric assert(Bundle.isFixed() && "Reached block before any predecessors"); 5190b57cec5SDimitry Andric 5200b57cec5SDimitry Andric // Push the fixed live-in registers. 5210b57cec5SDimitry Andric for (unsigned i = Bundle.FixCount; i > 0; --i) { 5220b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Live-in st(" << (i - 1) << "): %fp" 5230b57cec5SDimitry Andric << unsigned(Bundle.FixStack[i - 1]) << '\n'); 5240b57cec5SDimitry Andric pushReg(Bundle.FixStack[i-1]); 5250b57cec5SDimitry Andric } 5260b57cec5SDimitry Andric 5270b57cec5SDimitry Andric // Kill off unwanted live-ins. This can happen with a critical edge. 5280b57cec5SDimitry Andric // FIXME: We could keep these live registers around as zombies. They may need 5290b57cec5SDimitry Andric // to be revived at the end of a short block. It might save a few instrs. 5300b57cec5SDimitry Andric unsigned Mask = calcLiveInMask(MBB, /*RemoveFPs=*/true); 5310b57cec5SDimitry Andric adjustLiveRegs(Mask, MBB->begin()); 5320b57cec5SDimitry Andric LLVM_DEBUG(MBB->dump()); 5330b57cec5SDimitry Andric } 5340b57cec5SDimitry Andric 5350b57cec5SDimitry Andric /// finishBlockStack - Revive live-outs that are implicitly defined out of 5360b57cec5SDimitry Andric /// MBB. Shuffle live registers to match the expected fixed stack of any 5370b57cec5SDimitry Andric /// predecessors, and ensure that all predecessors are expecting the same 5380b57cec5SDimitry Andric /// stack. 5390b57cec5SDimitry Andric void FPS::finishBlockStack() { 5400b57cec5SDimitry Andric // The RET handling below takes care of return blocks for us. 5410b57cec5SDimitry Andric if (MBB->succ_empty()) 5420b57cec5SDimitry Andric return; 5430b57cec5SDimitry Andric 5440b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Setting up live-outs for " << printMBBReference(*MBB) 5450b57cec5SDimitry Andric << " derived from " << MBB->getName() << ".\n"); 5460b57cec5SDimitry Andric 5470b57cec5SDimitry Andric // Get MBB's live-out bundle. 5480b57cec5SDimitry Andric unsigned BundleIdx = Bundles->getBundle(MBB->getNumber(), true); 5490b57cec5SDimitry Andric LiveBundle &Bundle = LiveBundles[BundleIdx]; 5500b57cec5SDimitry Andric 5510b57cec5SDimitry Andric // We may need to kill and define some registers to match successors. 5520b57cec5SDimitry Andric // FIXME: This can probably be combined with the shuffle below. 5530b57cec5SDimitry Andric MachineBasicBlock::iterator Term = MBB->getFirstTerminator(); 5540b57cec5SDimitry Andric adjustLiveRegs(Bundle.Mask, Term); 5550b57cec5SDimitry Andric 5560b57cec5SDimitry Andric if (!Bundle.Mask) { 5570b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "No live-outs.\n"); 5580b57cec5SDimitry Andric return; 5590b57cec5SDimitry Andric } 5600b57cec5SDimitry Andric 5610b57cec5SDimitry Andric // Has the stack order been fixed yet? 5620b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "LB#" << BundleIdx << ": "); 5630b57cec5SDimitry Andric if (Bundle.isFixed()) { 5640b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Shuffling stack to match.\n"); 5650b57cec5SDimitry Andric shuffleStackTop(Bundle.FixStack, Bundle.FixCount, Term); 5660b57cec5SDimitry Andric } else { 5670b57cec5SDimitry Andric // Not fixed yet, we get to choose. 5680b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Fixing stack order now.\n"); 5690b57cec5SDimitry Andric Bundle.FixCount = StackTop; 5700b57cec5SDimitry Andric for (unsigned i = 0; i < StackTop; ++i) 5710b57cec5SDimitry Andric Bundle.FixStack[i] = getStackEntry(i); 5720b57cec5SDimitry Andric } 5730b57cec5SDimitry Andric } 5740b57cec5SDimitry Andric 5750b57cec5SDimitry Andric 5760b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 5770b57cec5SDimitry Andric // Efficient Lookup Table Support 5780b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 5790b57cec5SDimitry Andric 5800b57cec5SDimitry Andric namespace { 5810b57cec5SDimitry Andric struct TableEntry { 5820b57cec5SDimitry Andric uint16_t from; 5830b57cec5SDimitry Andric uint16_t to; 5840b57cec5SDimitry Andric bool operator<(const TableEntry &TE) const { return from < TE.from; } 5850b57cec5SDimitry Andric friend bool operator<(const TableEntry &TE, unsigned V) { 5860b57cec5SDimitry Andric return TE.from < V; 5870b57cec5SDimitry Andric } 5880b57cec5SDimitry Andric friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned V, 5890b57cec5SDimitry Andric const TableEntry &TE) { 5900b57cec5SDimitry Andric return V < TE.from; 5910b57cec5SDimitry Andric } 5920b57cec5SDimitry Andric }; 5930b57cec5SDimitry Andric } 5940b57cec5SDimitry Andric 5950b57cec5SDimitry Andric static int Lookup(ArrayRef<TableEntry> Table, unsigned Opcode) { 5960b57cec5SDimitry Andric const TableEntry *I = llvm::lower_bound(Table, Opcode); 5970b57cec5SDimitry Andric if (I != Table.end() && I->from == Opcode) 5980b57cec5SDimitry Andric return I->to; 5990b57cec5SDimitry Andric return -1; 6000b57cec5SDimitry Andric } 6010b57cec5SDimitry Andric 6020b57cec5SDimitry Andric #ifdef NDEBUG 6030b57cec5SDimitry Andric #define ASSERT_SORTED(TABLE) 6040b57cec5SDimitry Andric #else 6050b57cec5SDimitry Andric #define ASSERT_SORTED(TABLE) \ 6060b57cec5SDimitry Andric { \ 6070b57cec5SDimitry Andric static std::atomic<bool> TABLE##Checked(false); \ 6080b57cec5SDimitry Andric if (!TABLE##Checked.load(std::memory_order_relaxed)) { \ 609fe6060f1SDimitry Andric assert(is_sorted(TABLE) && \ 6100b57cec5SDimitry Andric "All lookup tables must be sorted for efficient access!"); \ 6110b57cec5SDimitry Andric TABLE##Checked.store(true, std::memory_order_relaxed); \ 6120b57cec5SDimitry Andric } \ 6130b57cec5SDimitry Andric } 6140b57cec5SDimitry Andric #endif 6150b57cec5SDimitry Andric 6160b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 6170b57cec5SDimitry Andric // Register File -> Register Stack Mapping Methods 6180b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 6190b57cec5SDimitry Andric 6200b57cec5SDimitry Andric // OpcodeTable - Sorted map of register instructions to their stack version. 6210b57cec5SDimitry Andric // The first element is an register file pseudo instruction, the second is the 6220b57cec5SDimitry Andric // concrete X86 instruction which uses the register stack. 6230b57cec5SDimitry Andric // 6240b57cec5SDimitry Andric static const TableEntry OpcodeTable[] = { 6250b57cec5SDimitry Andric { X86::ABS_Fp32 , X86::ABS_F }, 6260b57cec5SDimitry Andric { X86::ABS_Fp64 , X86::ABS_F }, 6270b57cec5SDimitry Andric { X86::ABS_Fp80 , X86::ABS_F }, 6280b57cec5SDimitry Andric { X86::ADD_Fp32m , X86::ADD_F32m }, 6290b57cec5SDimitry Andric { X86::ADD_Fp64m , X86::ADD_F64m }, 6300b57cec5SDimitry Andric { X86::ADD_Fp64m32 , X86::ADD_F32m }, 6310b57cec5SDimitry Andric { X86::ADD_Fp80m32 , X86::ADD_F32m }, 6320b57cec5SDimitry Andric { X86::ADD_Fp80m64 , X86::ADD_F64m }, 6330b57cec5SDimitry Andric { X86::ADD_FpI16m32 , X86::ADD_FI16m }, 6340b57cec5SDimitry Andric { X86::ADD_FpI16m64 , X86::ADD_FI16m }, 6350b57cec5SDimitry Andric { X86::ADD_FpI16m80 , X86::ADD_FI16m }, 6360b57cec5SDimitry Andric { X86::ADD_FpI32m32 , X86::ADD_FI32m }, 6370b57cec5SDimitry Andric { X86::ADD_FpI32m64 , X86::ADD_FI32m }, 6380b57cec5SDimitry Andric { X86::ADD_FpI32m80 , X86::ADD_FI32m }, 6390b57cec5SDimitry Andric { X86::CHS_Fp32 , X86::CHS_F }, 6400b57cec5SDimitry Andric { X86::CHS_Fp64 , X86::CHS_F }, 6410b57cec5SDimitry Andric { X86::CHS_Fp80 , X86::CHS_F }, 6420b57cec5SDimitry Andric { X86::CMOVBE_Fp32 , X86::CMOVBE_F }, 6430b57cec5SDimitry Andric { X86::CMOVBE_Fp64 , X86::CMOVBE_F }, 6440b57cec5SDimitry Andric { X86::CMOVBE_Fp80 , X86::CMOVBE_F }, 6450b57cec5SDimitry Andric { X86::CMOVB_Fp32 , X86::CMOVB_F }, 6460b57cec5SDimitry Andric { X86::CMOVB_Fp64 , X86::CMOVB_F }, 6470b57cec5SDimitry Andric { X86::CMOVB_Fp80 , X86::CMOVB_F }, 6480b57cec5SDimitry Andric { X86::CMOVE_Fp32 , X86::CMOVE_F }, 6490b57cec5SDimitry Andric { X86::CMOVE_Fp64 , X86::CMOVE_F }, 6500b57cec5SDimitry Andric { X86::CMOVE_Fp80 , X86::CMOVE_F }, 6510b57cec5SDimitry Andric { X86::CMOVNBE_Fp32 , X86::CMOVNBE_F }, 6520b57cec5SDimitry Andric { X86::CMOVNBE_Fp64 , X86::CMOVNBE_F }, 6530b57cec5SDimitry Andric { X86::CMOVNBE_Fp80 , X86::CMOVNBE_F }, 6540b57cec5SDimitry Andric { X86::CMOVNB_Fp32 , X86::CMOVNB_F }, 6550b57cec5SDimitry Andric { X86::CMOVNB_Fp64 , X86::CMOVNB_F }, 6560b57cec5SDimitry Andric { X86::CMOVNB_Fp80 , X86::CMOVNB_F }, 6570b57cec5SDimitry Andric { X86::CMOVNE_Fp32 , X86::CMOVNE_F }, 6580b57cec5SDimitry Andric { X86::CMOVNE_Fp64 , X86::CMOVNE_F }, 6590b57cec5SDimitry Andric { X86::CMOVNE_Fp80 , X86::CMOVNE_F }, 6600b57cec5SDimitry Andric { X86::CMOVNP_Fp32 , X86::CMOVNP_F }, 6610b57cec5SDimitry Andric { X86::CMOVNP_Fp64 , X86::CMOVNP_F }, 6620b57cec5SDimitry Andric { X86::CMOVNP_Fp80 , X86::CMOVNP_F }, 6630b57cec5SDimitry Andric { X86::CMOVP_Fp32 , X86::CMOVP_F }, 6640b57cec5SDimitry Andric { X86::CMOVP_Fp64 , X86::CMOVP_F }, 6650b57cec5SDimitry Andric { X86::CMOVP_Fp80 , X86::CMOVP_F }, 666480093f4SDimitry Andric { X86::COM_FpIr32 , X86::COM_FIr }, 667480093f4SDimitry Andric { X86::COM_FpIr64 , X86::COM_FIr }, 668480093f4SDimitry Andric { X86::COM_FpIr80 , X86::COM_FIr }, 669480093f4SDimitry Andric { X86::COM_Fpr32 , X86::COM_FST0r }, 670480093f4SDimitry Andric { X86::COM_Fpr64 , X86::COM_FST0r }, 671480093f4SDimitry Andric { X86::COM_Fpr80 , X86::COM_FST0r }, 6720b57cec5SDimitry Andric { X86::DIVR_Fp32m , X86::DIVR_F32m }, 6730b57cec5SDimitry Andric { X86::DIVR_Fp64m , X86::DIVR_F64m }, 6740b57cec5SDimitry Andric { X86::DIVR_Fp64m32 , X86::DIVR_F32m }, 6750b57cec5SDimitry Andric { X86::DIVR_Fp80m32 , X86::DIVR_F32m }, 6760b57cec5SDimitry Andric { X86::DIVR_Fp80m64 , X86::DIVR_F64m }, 6770b57cec5SDimitry Andric { X86::DIVR_FpI16m32, X86::DIVR_FI16m}, 6780b57cec5SDimitry Andric { X86::DIVR_FpI16m64, X86::DIVR_FI16m}, 6790b57cec5SDimitry Andric { X86::DIVR_FpI16m80, X86::DIVR_FI16m}, 6800b57cec5SDimitry Andric { X86::DIVR_FpI32m32, X86::DIVR_FI32m}, 6810b57cec5SDimitry Andric { X86::DIVR_FpI32m64, X86::DIVR_FI32m}, 6820b57cec5SDimitry Andric { X86::DIVR_FpI32m80, X86::DIVR_FI32m}, 6830b57cec5SDimitry Andric { X86::DIV_Fp32m , X86::DIV_F32m }, 6840b57cec5SDimitry Andric { X86::DIV_Fp64m , X86::DIV_F64m }, 6850b57cec5SDimitry Andric { X86::DIV_Fp64m32 , X86::DIV_F32m }, 6860b57cec5SDimitry Andric { X86::DIV_Fp80m32 , X86::DIV_F32m }, 6870b57cec5SDimitry Andric { X86::DIV_Fp80m64 , X86::DIV_F64m }, 6880b57cec5SDimitry Andric { X86::DIV_FpI16m32 , X86::DIV_FI16m }, 6890b57cec5SDimitry Andric { X86::DIV_FpI16m64 , X86::DIV_FI16m }, 6900b57cec5SDimitry Andric { X86::DIV_FpI16m80 , X86::DIV_FI16m }, 6910b57cec5SDimitry Andric { X86::DIV_FpI32m32 , X86::DIV_FI32m }, 6920b57cec5SDimitry Andric { X86::DIV_FpI32m64 , X86::DIV_FI32m }, 6930b57cec5SDimitry Andric { X86::DIV_FpI32m80 , X86::DIV_FI32m }, 6940b57cec5SDimitry Andric { X86::ILD_Fp16m32 , X86::ILD_F16m }, 6950b57cec5SDimitry Andric { X86::ILD_Fp16m64 , X86::ILD_F16m }, 6960b57cec5SDimitry Andric { X86::ILD_Fp16m80 , X86::ILD_F16m }, 6970b57cec5SDimitry Andric { X86::ILD_Fp32m32 , X86::ILD_F32m }, 6980b57cec5SDimitry Andric { X86::ILD_Fp32m64 , X86::ILD_F32m }, 6990b57cec5SDimitry Andric { X86::ILD_Fp32m80 , X86::ILD_F32m }, 7000b57cec5SDimitry Andric { X86::ILD_Fp64m32 , X86::ILD_F64m }, 7010b57cec5SDimitry Andric { X86::ILD_Fp64m64 , X86::ILD_F64m }, 7020b57cec5SDimitry Andric { X86::ILD_Fp64m80 , X86::ILD_F64m }, 7030b57cec5SDimitry Andric { X86::ISTT_Fp16m32 , X86::ISTT_FP16m}, 7040b57cec5SDimitry Andric { X86::ISTT_Fp16m64 , X86::ISTT_FP16m}, 7050b57cec5SDimitry Andric { X86::ISTT_Fp16m80 , X86::ISTT_FP16m}, 7060b57cec5SDimitry Andric { X86::ISTT_Fp32m32 , X86::ISTT_FP32m}, 7070b57cec5SDimitry Andric { X86::ISTT_Fp32m64 , X86::ISTT_FP32m}, 7080b57cec5SDimitry Andric { X86::ISTT_Fp32m80 , X86::ISTT_FP32m}, 7090b57cec5SDimitry Andric { X86::ISTT_Fp64m32 , X86::ISTT_FP64m}, 7100b57cec5SDimitry Andric { X86::ISTT_Fp64m64 , X86::ISTT_FP64m}, 7110b57cec5SDimitry Andric { X86::ISTT_Fp64m80 , X86::ISTT_FP64m}, 7120b57cec5SDimitry Andric { X86::IST_Fp16m32 , X86::IST_F16m }, 7130b57cec5SDimitry Andric { X86::IST_Fp16m64 , X86::IST_F16m }, 7140b57cec5SDimitry Andric { X86::IST_Fp16m80 , X86::IST_F16m }, 7150b57cec5SDimitry Andric { X86::IST_Fp32m32 , X86::IST_F32m }, 7160b57cec5SDimitry Andric { X86::IST_Fp32m64 , X86::IST_F32m }, 7170b57cec5SDimitry Andric { X86::IST_Fp32m80 , X86::IST_F32m }, 7180b57cec5SDimitry Andric { X86::IST_Fp64m32 , X86::IST_FP64m }, 7190b57cec5SDimitry Andric { X86::IST_Fp64m64 , X86::IST_FP64m }, 7200b57cec5SDimitry Andric { X86::IST_Fp64m80 , X86::IST_FP64m }, 7210b57cec5SDimitry Andric { X86::LD_Fp032 , X86::LD_F0 }, 7220b57cec5SDimitry Andric { X86::LD_Fp064 , X86::LD_F0 }, 7230b57cec5SDimitry Andric { X86::LD_Fp080 , X86::LD_F0 }, 7240b57cec5SDimitry Andric { X86::LD_Fp132 , X86::LD_F1 }, 7250b57cec5SDimitry Andric { X86::LD_Fp164 , X86::LD_F1 }, 7260b57cec5SDimitry Andric { X86::LD_Fp180 , X86::LD_F1 }, 7270b57cec5SDimitry Andric { X86::LD_Fp32m , X86::LD_F32m }, 7280b57cec5SDimitry Andric { X86::LD_Fp32m64 , X86::LD_F32m }, 7290b57cec5SDimitry Andric { X86::LD_Fp32m80 , X86::LD_F32m }, 7300b57cec5SDimitry Andric { X86::LD_Fp64m , X86::LD_F64m }, 7310b57cec5SDimitry Andric { X86::LD_Fp64m80 , X86::LD_F64m }, 7320b57cec5SDimitry Andric { X86::LD_Fp80m , X86::LD_F80m }, 7330b57cec5SDimitry Andric { X86::MUL_Fp32m , X86::MUL_F32m }, 7340b57cec5SDimitry Andric { X86::MUL_Fp64m , X86::MUL_F64m }, 7350b57cec5SDimitry Andric { X86::MUL_Fp64m32 , X86::MUL_F32m }, 7360b57cec5SDimitry Andric { X86::MUL_Fp80m32 , X86::MUL_F32m }, 7370b57cec5SDimitry Andric { X86::MUL_Fp80m64 , X86::MUL_F64m }, 7380b57cec5SDimitry Andric { X86::MUL_FpI16m32 , X86::MUL_FI16m }, 7390b57cec5SDimitry Andric { X86::MUL_FpI16m64 , X86::MUL_FI16m }, 7400b57cec5SDimitry Andric { X86::MUL_FpI16m80 , X86::MUL_FI16m }, 7410b57cec5SDimitry Andric { X86::MUL_FpI32m32 , X86::MUL_FI32m }, 7420b57cec5SDimitry Andric { X86::MUL_FpI32m64 , X86::MUL_FI32m }, 7430b57cec5SDimitry Andric { X86::MUL_FpI32m80 , X86::MUL_FI32m }, 7440b57cec5SDimitry Andric { X86::SQRT_Fp32 , X86::SQRT_F }, 7450b57cec5SDimitry Andric { X86::SQRT_Fp64 , X86::SQRT_F }, 7460b57cec5SDimitry Andric { X86::SQRT_Fp80 , X86::SQRT_F }, 7470b57cec5SDimitry Andric { X86::ST_Fp32m , X86::ST_F32m }, 7480b57cec5SDimitry Andric { X86::ST_Fp64m , X86::ST_F64m }, 7490b57cec5SDimitry Andric { X86::ST_Fp64m32 , X86::ST_F32m }, 7500b57cec5SDimitry Andric { X86::ST_Fp80m32 , X86::ST_F32m }, 7510b57cec5SDimitry Andric { X86::ST_Fp80m64 , X86::ST_F64m }, 7520b57cec5SDimitry Andric { X86::ST_FpP80m , X86::ST_FP80m }, 7530b57cec5SDimitry Andric { X86::SUBR_Fp32m , X86::SUBR_F32m }, 7540b57cec5SDimitry Andric { X86::SUBR_Fp64m , X86::SUBR_F64m }, 7550b57cec5SDimitry Andric { X86::SUBR_Fp64m32 , X86::SUBR_F32m }, 7560b57cec5SDimitry Andric { X86::SUBR_Fp80m32 , X86::SUBR_F32m }, 7570b57cec5SDimitry Andric { X86::SUBR_Fp80m64 , X86::SUBR_F64m }, 7580b57cec5SDimitry Andric { X86::SUBR_FpI16m32, X86::SUBR_FI16m}, 7590b57cec5SDimitry Andric { X86::SUBR_FpI16m64, X86::SUBR_FI16m}, 7600b57cec5SDimitry Andric { X86::SUBR_FpI16m80, X86::SUBR_FI16m}, 7610b57cec5SDimitry Andric { X86::SUBR_FpI32m32, X86::SUBR_FI32m}, 7620b57cec5SDimitry Andric { X86::SUBR_FpI32m64, X86::SUBR_FI32m}, 7630b57cec5SDimitry Andric { X86::SUBR_FpI32m80, X86::SUBR_FI32m}, 7640b57cec5SDimitry Andric { X86::SUB_Fp32m , X86::SUB_F32m }, 7650b57cec5SDimitry Andric { X86::SUB_Fp64m , X86::SUB_F64m }, 7660b57cec5SDimitry Andric { X86::SUB_Fp64m32 , X86::SUB_F32m }, 7670b57cec5SDimitry Andric { X86::SUB_Fp80m32 , X86::SUB_F32m }, 7680b57cec5SDimitry Andric { X86::SUB_Fp80m64 , X86::SUB_F64m }, 7690b57cec5SDimitry Andric { X86::SUB_FpI16m32 , X86::SUB_FI16m }, 7700b57cec5SDimitry Andric { X86::SUB_FpI16m64 , X86::SUB_FI16m }, 7710b57cec5SDimitry Andric { X86::SUB_FpI16m80 , X86::SUB_FI16m }, 7720b57cec5SDimitry Andric { X86::SUB_FpI32m32 , X86::SUB_FI32m }, 7730b57cec5SDimitry Andric { X86::SUB_FpI32m64 , X86::SUB_FI32m }, 7740b57cec5SDimitry Andric { X86::SUB_FpI32m80 , X86::SUB_FI32m }, 7750b57cec5SDimitry Andric { X86::TST_Fp32 , X86::TST_F }, 7760b57cec5SDimitry Andric { X86::TST_Fp64 , X86::TST_F }, 7770b57cec5SDimitry Andric { X86::TST_Fp80 , X86::TST_F }, 7780b57cec5SDimitry Andric { X86::UCOM_FpIr32 , X86::UCOM_FIr }, 7790b57cec5SDimitry Andric { X86::UCOM_FpIr64 , X86::UCOM_FIr }, 7800b57cec5SDimitry Andric { X86::UCOM_FpIr80 , X86::UCOM_FIr }, 7810b57cec5SDimitry Andric { X86::UCOM_Fpr32 , X86::UCOM_Fr }, 7820b57cec5SDimitry Andric { X86::UCOM_Fpr64 , X86::UCOM_Fr }, 7830b57cec5SDimitry Andric { X86::UCOM_Fpr80 , X86::UCOM_Fr }, 784fe6060f1SDimitry Andric { X86::XAM_Fp32 , X86::XAM_F }, 785fe6060f1SDimitry Andric { X86::XAM_Fp64 , X86::XAM_F }, 786fe6060f1SDimitry Andric { X86::XAM_Fp80 , X86::XAM_F }, 7870b57cec5SDimitry Andric }; 7880b57cec5SDimitry Andric 7890b57cec5SDimitry Andric static unsigned getConcreteOpcode(unsigned Opcode) { 7900b57cec5SDimitry Andric ASSERT_SORTED(OpcodeTable); 7910b57cec5SDimitry Andric int Opc = Lookup(OpcodeTable, Opcode); 7920b57cec5SDimitry Andric assert(Opc != -1 && "FP Stack instruction not in OpcodeTable!"); 7930b57cec5SDimitry Andric return Opc; 7940b57cec5SDimitry Andric } 7950b57cec5SDimitry Andric 7960b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 7970b57cec5SDimitry Andric // Helper Methods 7980b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 7990b57cec5SDimitry Andric 8000b57cec5SDimitry Andric // PopTable - Sorted map of instructions to their popping version. The first 8010b57cec5SDimitry Andric // element is an instruction, the second is the version which pops. 8020b57cec5SDimitry Andric // 8030b57cec5SDimitry Andric static const TableEntry PopTable[] = { 8040b57cec5SDimitry Andric { X86::ADD_FrST0 , X86::ADD_FPrST0 }, 8050b57cec5SDimitry Andric 806480093f4SDimitry Andric { X86::COMP_FST0r, X86::FCOMPP }, 807480093f4SDimitry Andric { X86::COM_FIr , X86::COM_FIPr }, 808480093f4SDimitry Andric { X86::COM_FST0r , X86::COMP_FST0r }, 809480093f4SDimitry Andric 8100b57cec5SDimitry Andric { X86::DIVR_FrST0, X86::DIVR_FPrST0 }, 8110b57cec5SDimitry Andric { X86::DIV_FrST0 , X86::DIV_FPrST0 }, 8120b57cec5SDimitry Andric 8130b57cec5SDimitry Andric { X86::IST_F16m , X86::IST_FP16m }, 8140b57cec5SDimitry Andric { X86::IST_F32m , X86::IST_FP32m }, 8150b57cec5SDimitry Andric 8160b57cec5SDimitry Andric { X86::MUL_FrST0 , X86::MUL_FPrST0 }, 8170b57cec5SDimitry Andric 8180b57cec5SDimitry Andric { X86::ST_F32m , X86::ST_FP32m }, 8190b57cec5SDimitry Andric { X86::ST_F64m , X86::ST_FP64m }, 8200b57cec5SDimitry Andric { X86::ST_Frr , X86::ST_FPrr }, 8210b57cec5SDimitry Andric 8220b57cec5SDimitry Andric { X86::SUBR_FrST0, X86::SUBR_FPrST0 }, 8230b57cec5SDimitry Andric { X86::SUB_FrST0 , X86::SUB_FPrST0 }, 8240b57cec5SDimitry Andric 8250b57cec5SDimitry Andric { X86::UCOM_FIr , X86::UCOM_FIPr }, 8260b57cec5SDimitry Andric 8270b57cec5SDimitry Andric { X86::UCOM_FPr , X86::UCOM_FPPr }, 8280b57cec5SDimitry Andric { X86::UCOM_Fr , X86::UCOM_FPr }, 8290b57cec5SDimitry Andric }; 8300b57cec5SDimitry Andric 831349cc55cSDimitry Andric static bool doesInstructionSetFPSW(MachineInstr &MI) { 832*0fca6ea1SDimitry Andric if (const MachineOperand *MO = 833*0fca6ea1SDimitry Andric MI.findRegisterDefOperand(X86::FPSW, /*TRI=*/nullptr)) 834349cc55cSDimitry Andric if (!MO->isDead()) 835349cc55cSDimitry Andric return true; 836349cc55cSDimitry Andric return false; 837349cc55cSDimitry Andric } 838349cc55cSDimitry Andric 839349cc55cSDimitry Andric static MachineBasicBlock::iterator 840349cc55cSDimitry Andric getNextFPInstruction(MachineBasicBlock::iterator I) { 841349cc55cSDimitry Andric MachineBasicBlock &MBB = *I->getParent(); 842349cc55cSDimitry Andric while (++I != MBB.end()) { 843349cc55cSDimitry Andric MachineInstr &MI = *I; 844349cc55cSDimitry Andric if (X86::isX87Instruction(MI)) 845349cc55cSDimitry Andric return I; 846349cc55cSDimitry Andric } 847349cc55cSDimitry Andric return MBB.end(); 848349cc55cSDimitry Andric } 849349cc55cSDimitry Andric 8500b57cec5SDimitry Andric /// popStackAfter - Pop the current value off of the top of the FP stack after 8510b57cec5SDimitry Andric /// the specified instruction. This attempts to be sneaky and combine the pop 8520b57cec5SDimitry Andric /// into the instruction itself if possible. The iterator is left pointing to 8530b57cec5SDimitry Andric /// the last instruction, be it a new pop instruction inserted, or the old 8540b57cec5SDimitry Andric /// instruction if it was modified in place. 8550b57cec5SDimitry Andric /// 8560b57cec5SDimitry Andric void FPS::popStackAfter(MachineBasicBlock::iterator &I) { 8570b57cec5SDimitry Andric MachineInstr &MI = *I; 8580b57cec5SDimitry Andric const DebugLoc &dl = MI.getDebugLoc(); 8590b57cec5SDimitry Andric ASSERT_SORTED(PopTable); 8600b57cec5SDimitry Andric 8610b57cec5SDimitry Andric popReg(); 8620b57cec5SDimitry Andric 8630b57cec5SDimitry Andric // Check to see if there is a popping version of this instruction... 8640b57cec5SDimitry Andric int Opcode = Lookup(PopTable, I->getOpcode()); 8650b57cec5SDimitry Andric if (Opcode != -1) { 8660b57cec5SDimitry Andric I->setDesc(TII->get(Opcode)); 867480093f4SDimitry Andric if (Opcode == X86::FCOMPP || Opcode == X86::UCOM_FPPr) 86881ad6265SDimitry Andric I->removeOperand(0); 869fe6060f1SDimitry Andric MI.dropDebugNumber(); 8700b57cec5SDimitry Andric } else { // Insert an explicit pop 871349cc55cSDimitry Andric // If this instruction sets FPSW, which is read in following instruction, 872349cc55cSDimitry Andric // insert pop after that reader. 873349cc55cSDimitry Andric if (doesInstructionSetFPSW(MI)) { 874349cc55cSDimitry Andric MachineBasicBlock &MBB = *MI.getParent(); 875349cc55cSDimitry Andric MachineBasicBlock::iterator Next = getNextFPInstruction(I); 876*0fca6ea1SDimitry Andric if (Next != MBB.end() && Next->readsRegister(X86::FPSW, /*TRI=*/nullptr)) 877349cc55cSDimitry Andric I = Next; 878349cc55cSDimitry Andric } 8790b57cec5SDimitry Andric I = BuildMI(*MBB, ++I, dl, TII->get(X86::ST_FPrr)).addReg(X86::ST0); 8800b57cec5SDimitry Andric } 8810b57cec5SDimitry Andric } 8820b57cec5SDimitry Andric 8830b57cec5SDimitry Andric /// freeStackSlotAfter - Free the specified register from the register stack, so 8840b57cec5SDimitry Andric /// that it is no longer in a register. If the register is currently at the top 8850b57cec5SDimitry Andric /// of the stack, we just pop the current instruction, otherwise we store the 8860b57cec5SDimitry Andric /// current top-of-stack into the specified slot, then pop the top of stack. 8870b57cec5SDimitry Andric void FPS::freeStackSlotAfter(MachineBasicBlock::iterator &I, unsigned FPRegNo) { 8880b57cec5SDimitry Andric if (getStackEntry(0) == FPRegNo) { // already at the top of stack? easy. 8890b57cec5SDimitry Andric popStackAfter(I); 8900b57cec5SDimitry Andric return; 8910b57cec5SDimitry Andric } 8920b57cec5SDimitry Andric 8930b57cec5SDimitry Andric // Otherwise, store the top of stack into the dead slot, killing the operand 8940b57cec5SDimitry Andric // without having to add in an explicit xchg then pop. 8950b57cec5SDimitry Andric // 8960b57cec5SDimitry Andric I = freeStackSlotBefore(++I, FPRegNo); 8970b57cec5SDimitry Andric } 8980b57cec5SDimitry Andric 8990b57cec5SDimitry Andric /// freeStackSlotBefore - Free the specified register without trying any 9000b57cec5SDimitry Andric /// folding. 9010b57cec5SDimitry Andric MachineBasicBlock::iterator 9020b57cec5SDimitry Andric FPS::freeStackSlotBefore(MachineBasicBlock::iterator I, unsigned FPRegNo) { 9030b57cec5SDimitry Andric unsigned STReg = getSTReg(FPRegNo); 9040b57cec5SDimitry Andric unsigned OldSlot = getSlot(FPRegNo); 9050b57cec5SDimitry Andric unsigned TopReg = Stack[StackTop-1]; 9060b57cec5SDimitry Andric Stack[OldSlot] = TopReg; 9070b57cec5SDimitry Andric RegMap[TopReg] = OldSlot; 9080b57cec5SDimitry Andric RegMap[FPRegNo] = ~0; 9090b57cec5SDimitry Andric Stack[--StackTop] = ~0; 9100b57cec5SDimitry Andric return BuildMI(*MBB, I, DebugLoc(), TII->get(X86::ST_FPrr)) 9110b57cec5SDimitry Andric .addReg(STReg) 9120b57cec5SDimitry Andric .getInstr(); 9130b57cec5SDimitry Andric } 9140b57cec5SDimitry Andric 9150b57cec5SDimitry Andric /// adjustLiveRegs - Kill and revive registers such that exactly the FP 9160b57cec5SDimitry Andric /// registers with a bit in Mask are live. 9170b57cec5SDimitry Andric void FPS::adjustLiveRegs(unsigned Mask, MachineBasicBlock::iterator I) { 9180b57cec5SDimitry Andric unsigned Defs = Mask; 9190b57cec5SDimitry Andric unsigned Kills = 0; 9200b57cec5SDimitry Andric for (unsigned i = 0; i < StackTop; ++i) { 9210b57cec5SDimitry Andric unsigned RegNo = Stack[i]; 9220b57cec5SDimitry Andric if (!(Defs & (1 << RegNo))) 9230b57cec5SDimitry Andric // This register is live, but we don't want it. 9240b57cec5SDimitry Andric Kills |= (1 << RegNo); 9250b57cec5SDimitry Andric else 9260b57cec5SDimitry Andric // We don't need to imp-def this live register. 9270b57cec5SDimitry Andric Defs &= ~(1 << RegNo); 9280b57cec5SDimitry Andric } 9290b57cec5SDimitry Andric assert((Kills & Defs) == 0 && "Register needs killing and def'ing?"); 9300b57cec5SDimitry Andric 9310b57cec5SDimitry Andric // Produce implicit-defs for free by using killed registers. 9320b57cec5SDimitry Andric while (Kills && Defs) { 93306c3fb27SDimitry Andric unsigned KReg = llvm::countr_zero(Kills); 93406c3fb27SDimitry Andric unsigned DReg = llvm::countr_zero(Defs); 9350b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Renaming %fp" << KReg << " as imp %fp" << DReg 9360b57cec5SDimitry Andric << "\n"); 9370b57cec5SDimitry Andric std::swap(Stack[getSlot(KReg)], Stack[getSlot(DReg)]); 9380b57cec5SDimitry Andric std::swap(RegMap[KReg], RegMap[DReg]); 9390b57cec5SDimitry Andric Kills &= ~(1 << KReg); 9400b57cec5SDimitry Andric Defs &= ~(1 << DReg); 9410b57cec5SDimitry Andric } 9420b57cec5SDimitry Andric 9430b57cec5SDimitry Andric // Kill registers by popping. 9440b57cec5SDimitry Andric if (Kills && I != MBB->begin()) { 9450b57cec5SDimitry Andric MachineBasicBlock::iterator I2 = std::prev(I); 9460b57cec5SDimitry Andric while (StackTop) { 9470b57cec5SDimitry Andric unsigned KReg = getStackEntry(0); 9480b57cec5SDimitry Andric if (!(Kills & (1 << KReg))) 9490b57cec5SDimitry Andric break; 9500b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Popping %fp" << KReg << "\n"); 9510b57cec5SDimitry Andric popStackAfter(I2); 9520b57cec5SDimitry Andric Kills &= ~(1 << KReg); 9530b57cec5SDimitry Andric } 9540b57cec5SDimitry Andric } 9550b57cec5SDimitry Andric 9560b57cec5SDimitry Andric // Manually kill the rest. 9570b57cec5SDimitry Andric while (Kills) { 95806c3fb27SDimitry Andric unsigned KReg = llvm::countr_zero(Kills); 9590b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Killing %fp" << KReg << "\n"); 9600b57cec5SDimitry Andric freeStackSlotBefore(I, KReg); 9610b57cec5SDimitry Andric Kills &= ~(1 << KReg); 9620b57cec5SDimitry Andric } 9630b57cec5SDimitry Andric 9640b57cec5SDimitry Andric // Load zeros for all the imp-defs. 9650b57cec5SDimitry Andric while(Defs) { 96606c3fb27SDimitry Andric unsigned DReg = llvm::countr_zero(Defs); 9670b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Defining %fp" << DReg << " as 0\n"); 9680b57cec5SDimitry Andric BuildMI(*MBB, I, DebugLoc(), TII->get(X86::LD_F0)); 9690b57cec5SDimitry Andric pushReg(DReg); 9700b57cec5SDimitry Andric Defs &= ~(1 << DReg); 9710b57cec5SDimitry Andric } 9720b57cec5SDimitry Andric 9730b57cec5SDimitry Andric // Now we should have the correct registers live. 9740b57cec5SDimitry Andric LLVM_DEBUG(dumpStack()); 975bdd1243dSDimitry Andric assert(StackTop == (unsigned)llvm::popcount(Mask) && "Live count mismatch"); 9760b57cec5SDimitry Andric } 9770b57cec5SDimitry Andric 9780b57cec5SDimitry Andric /// shuffleStackTop - emit fxch instructions before I to shuffle the top 9790b57cec5SDimitry Andric /// FixCount entries into the order given by FixStack. 9800b57cec5SDimitry Andric /// FIXME: Is there a better algorithm than insertion sort? 9810b57cec5SDimitry Andric void FPS::shuffleStackTop(const unsigned char *FixStack, 9820b57cec5SDimitry Andric unsigned FixCount, 9830b57cec5SDimitry Andric MachineBasicBlock::iterator I) { 9840b57cec5SDimitry Andric // Move items into place, starting from the desired stack bottom. 9850b57cec5SDimitry Andric while (FixCount--) { 9860b57cec5SDimitry Andric // Old register at position FixCount. 9870b57cec5SDimitry Andric unsigned OldReg = getStackEntry(FixCount); 9880b57cec5SDimitry Andric // Desired register at position FixCount. 9890b57cec5SDimitry Andric unsigned Reg = FixStack[FixCount]; 9900b57cec5SDimitry Andric if (Reg == OldReg) 9910b57cec5SDimitry Andric continue; 9920b57cec5SDimitry Andric // (Reg st0) (OldReg st0) = (Reg OldReg st0) 9930b57cec5SDimitry Andric moveToTop(Reg, I); 9940b57cec5SDimitry Andric if (FixCount > 0) 9950b57cec5SDimitry Andric moveToTop(OldReg, I); 9960b57cec5SDimitry Andric } 9970b57cec5SDimitry Andric LLVM_DEBUG(dumpStack()); 9980b57cec5SDimitry Andric } 9990b57cec5SDimitry Andric 10000b57cec5SDimitry Andric 10010b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 10020b57cec5SDimitry Andric // Instruction transformation implementation 10030b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 10040b57cec5SDimitry Andric 10050b57cec5SDimitry Andric void FPS::handleCall(MachineBasicBlock::iterator &I) { 1006480093f4SDimitry Andric MachineInstr &MI = *I; 10070b57cec5SDimitry Andric unsigned STReturns = 0; 10080b57cec5SDimitry Andric 1009fe6060f1SDimitry Andric bool ClobbersFPStack = false; 1010480093f4SDimitry Andric for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 1011480093f4SDimitry Andric MachineOperand &Op = MI.getOperand(i); 1012fe6060f1SDimitry Andric // Check if this call clobbers the FP stack. 1013fe6060f1SDimitry Andric // is sufficient. 1014fe6060f1SDimitry Andric if (Op.isRegMask()) { 1015fe6060f1SDimitry Andric bool ClobbersFP0 = Op.clobbersPhysReg(X86::FP0); 1016fe6060f1SDimitry Andric #ifndef NDEBUG 1017fe6060f1SDimitry Andric static_assert(X86::FP7 - X86::FP0 == 7, "sequential FP regnumbers"); 1018fe6060f1SDimitry Andric for (unsigned i = 1; i != 8; ++i) 1019fe6060f1SDimitry Andric assert(Op.clobbersPhysReg(X86::FP0 + i) == ClobbersFP0 && 1020fe6060f1SDimitry Andric "Inconsistent FP register clobber"); 1021fe6060f1SDimitry Andric #endif 1022fe6060f1SDimitry Andric 1023fe6060f1SDimitry Andric if (ClobbersFP0) 1024fe6060f1SDimitry Andric ClobbersFPStack = true; 1025fe6060f1SDimitry Andric } 1026fe6060f1SDimitry Andric 1027480093f4SDimitry Andric if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6) 10280b57cec5SDimitry Andric continue; 10290b57cec5SDimitry Andric 1030480093f4SDimitry Andric assert(Op.isImplicit() && "Expected implicit def/use"); 10310b57cec5SDimitry Andric 1032480093f4SDimitry Andric if (Op.isDef()) 1033480093f4SDimitry Andric STReturns |= 1 << getFPReg(Op); 10340b57cec5SDimitry Andric 1035480093f4SDimitry Andric // Remove the operand so that later passes don't see it. 103681ad6265SDimitry Andric MI.removeOperand(i); 1037480093f4SDimitry Andric --i; 1038480093f4SDimitry Andric --e; 10390b57cec5SDimitry Andric } 10400b57cec5SDimitry Andric 1041fe6060f1SDimitry Andric // Most calls should have a regmask that clobbers the FP registers. If it 1042fe6060f1SDimitry Andric // isn't present then the register allocator didn't spill the FP registers 1043fe6060f1SDimitry Andric // so they are still on the stack. 1044fe6060f1SDimitry Andric assert((ClobbersFPStack || STReturns == 0) && 1045fe6060f1SDimitry Andric "ST returns without FP stack clobber"); 1046fe6060f1SDimitry Andric if (!ClobbersFPStack) 1047fe6060f1SDimitry Andric return; 1048fe6060f1SDimitry Andric 104906c3fb27SDimitry Andric unsigned N = llvm::countr_one(STReturns); 10500b57cec5SDimitry Andric 10510b57cec5SDimitry Andric // FP registers used for function return must be consecutive starting at 10520b57cec5SDimitry Andric // FP0 10530b57cec5SDimitry Andric assert(STReturns == 0 || (isMask_32(STReturns) && N <= 2)); 10540b57cec5SDimitry Andric 10550b57cec5SDimitry Andric // Reset the FP Stack - It is required because of possible leftovers from 10560b57cec5SDimitry Andric // passed arguments. The caller should assume that the FP stack is 10570b57cec5SDimitry Andric // returned empty (unless the callee returns values on FP stack). 10580b57cec5SDimitry Andric while (StackTop > 0) 10590b57cec5SDimitry Andric popReg(); 10600b57cec5SDimitry Andric 10610b57cec5SDimitry Andric for (unsigned I = 0; I < N; ++I) 10620b57cec5SDimitry Andric pushReg(N - I - 1); 1063fe6060f1SDimitry Andric 1064349cc55cSDimitry Andric // If this call has been modified, drop all variable values defined by it. 1065349cc55cSDimitry Andric // We can't track them once they've been stackified. 1066349cc55cSDimitry Andric if (STReturns) 1067fe6060f1SDimitry Andric I->dropDebugNumber(); 10680b57cec5SDimitry Andric } 10690b57cec5SDimitry Andric 10700b57cec5SDimitry Andric /// If RET has an FP register use operand, pass the first one in ST(0) and 10710b57cec5SDimitry Andric /// the second one in ST(1). 10720b57cec5SDimitry Andric void FPS::handleReturn(MachineBasicBlock::iterator &I) { 10730b57cec5SDimitry Andric MachineInstr &MI = *I; 10740b57cec5SDimitry Andric 10750b57cec5SDimitry Andric // Find the register operands. 10760b57cec5SDimitry Andric unsigned FirstFPRegOp = ~0U, SecondFPRegOp = ~0U; 10770b57cec5SDimitry Andric unsigned LiveMask = 0; 10780b57cec5SDimitry Andric 10790b57cec5SDimitry Andric for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 10800b57cec5SDimitry Andric MachineOperand &Op = MI.getOperand(i); 10810b57cec5SDimitry Andric if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6) 10820b57cec5SDimitry Andric continue; 10830b57cec5SDimitry Andric // FP Register uses must be kills unless there are two uses of the same 10840b57cec5SDimitry Andric // register, in which case only one will be a kill. 10850b57cec5SDimitry Andric assert(Op.isUse() && 10860b57cec5SDimitry Andric (Op.isKill() || // Marked kill. 10870b57cec5SDimitry Andric getFPReg(Op) == FirstFPRegOp || // Second instance. 1088*0fca6ea1SDimitry Andric MI.killsRegister(Op.getReg(), 1089*0fca6ea1SDimitry Andric /*TRI=*/nullptr)) && // Later use is marked kill. 10900b57cec5SDimitry Andric "Ret only defs operands, and values aren't live beyond it"); 10910b57cec5SDimitry Andric 10920b57cec5SDimitry Andric if (FirstFPRegOp == ~0U) 10930b57cec5SDimitry Andric FirstFPRegOp = getFPReg(Op); 10940b57cec5SDimitry Andric else { 10950b57cec5SDimitry Andric assert(SecondFPRegOp == ~0U && "More than two fp operands!"); 10960b57cec5SDimitry Andric SecondFPRegOp = getFPReg(Op); 10970b57cec5SDimitry Andric } 10980b57cec5SDimitry Andric LiveMask |= (1 << getFPReg(Op)); 10990b57cec5SDimitry Andric 11000b57cec5SDimitry Andric // Remove the operand so that later passes don't see it. 110181ad6265SDimitry Andric MI.removeOperand(i); 11020b57cec5SDimitry Andric --i; 11030b57cec5SDimitry Andric --e; 11040b57cec5SDimitry Andric } 11050b57cec5SDimitry Andric 11060b57cec5SDimitry Andric // We may have been carrying spurious live-ins, so make sure only the 11070b57cec5SDimitry Andric // returned registers are left live. 11080b57cec5SDimitry Andric adjustLiveRegs(LiveMask, MI); 11090b57cec5SDimitry Andric if (!LiveMask) return; // Quick check to see if any are possible. 11100b57cec5SDimitry Andric 11110b57cec5SDimitry Andric // There are only four possibilities here: 11120b57cec5SDimitry Andric // 1) we are returning a single FP value. In this case, it has to be in 11130b57cec5SDimitry Andric // ST(0) already, so just declare success by removing the value from the 11140b57cec5SDimitry Andric // FP Stack. 11150b57cec5SDimitry Andric if (SecondFPRegOp == ~0U) { 11160b57cec5SDimitry Andric // Assert that the top of stack contains the right FP register. 11170b57cec5SDimitry Andric assert(StackTop == 1 && FirstFPRegOp == getStackEntry(0) && 11180b57cec5SDimitry Andric "Top of stack not the right register for RET!"); 11190b57cec5SDimitry Andric 11200b57cec5SDimitry Andric // Ok, everything is good, mark the value as not being on the stack 11210b57cec5SDimitry Andric // anymore so that our assertion about the stack being empty at end of 11220b57cec5SDimitry Andric // block doesn't fire. 11230b57cec5SDimitry Andric StackTop = 0; 11240b57cec5SDimitry Andric return; 11250b57cec5SDimitry Andric } 11260b57cec5SDimitry Andric 11270b57cec5SDimitry Andric // Otherwise, we are returning two values: 11280b57cec5SDimitry Andric // 2) If returning the same value for both, we only have one thing in the FP 11290b57cec5SDimitry Andric // stack. Consider: RET FP1, FP1 11300b57cec5SDimitry Andric if (StackTop == 1) { 11310b57cec5SDimitry Andric assert(FirstFPRegOp == SecondFPRegOp && FirstFPRegOp == getStackEntry(0)&& 11320b57cec5SDimitry Andric "Stack misconfiguration for RET!"); 11330b57cec5SDimitry Andric 11340b57cec5SDimitry Andric // Duplicate the TOS so that we return it twice. Just pick some other FPx 11350b57cec5SDimitry Andric // register to hold it. 11360b57cec5SDimitry Andric unsigned NewReg = ScratchFPReg; 11370b57cec5SDimitry Andric duplicateToTop(FirstFPRegOp, NewReg, MI); 11380b57cec5SDimitry Andric FirstFPRegOp = NewReg; 11390b57cec5SDimitry Andric } 11400b57cec5SDimitry Andric 11410b57cec5SDimitry Andric /// Okay we know we have two different FPx operands now: 11420b57cec5SDimitry Andric assert(StackTop == 2 && "Must have two values live!"); 11430b57cec5SDimitry Andric 11440b57cec5SDimitry Andric /// 3) If SecondFPRegOp is currently in ST(0) and FirstFPRegOp is currently 11450b57cec5SDimitry Andric /// in ST(1). In this case, emit an fxch. 11460b57cec5SDimitry Andric if (getStackEntry(0) == SecondFPRegOp) { 11470b57cec5SDimitry Andric assert(getStackEntry(1) == FirstFPRegOp && "Unknown regs live"); 11480b57cec5SDimitry Andric moveToTop(FirstFPRegOp, MI); 11490b57cec5SDimitry Andric } 11500b57cec5SDimitry Andric 11510b57cec5SDimitry Andric /// 4) Finally, FirstFPRegOp must be in ST(0) and SecondFPRegOp must be in 11520b57cec5SDimitry Andric /// ST(1). Just remove both from our understanding of the stack and return. 11530b57cec5SDimitry Andric assert(getStackEntry(0) == FirstFPRegOp && "Unknown regs live"); 11540b57cec5SDimitry Andric assert(getStackEntry(1) == SecondFPRegOp && "Unknown regs live"); 11550b57cec5SDimitry Andric StackTop = 0; 11560b57cec5SDimitry Andric } 11570b57cec5SDimitry Andric 11580b57cec5SDimitry Andric /// handleZeroArgFP - ST(0) = fld0 ST(0) = flds <mem> 11590b57cec5SDimitry Andric /// 11600b57cec5SDimitry Andric void FPS::handleZeroArgFP(MachineBasicBlock::iterator &I) { 11610b57cec5SDimitry Andric MachineInstr &MI = *I; 11620b57cec5SDimitry Andric unsigned DestReg = getFPReg(MI.getOperand(0)); 11630b57cec5SDimitry Andric 11640b57cec5SDimitry Andric // Change from the pseudo instruction to the concrete instruction. 116581ad6265SDimitry Andric MI.removeOperand(0); // Remove the explicit ST(0) operand 11660b57cec5SDimitry Andric MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode()))); 11670b57cec5SDimitry Andric MI.addOperand( 11680b57cec5SDimitry Andric MachineOperand::CreateReg(X86::ST0, /*isDef*/ true, /*isImp*/ true)); 11690b57cec5SDimitry Andric 11700b57cec5SDimitry Andric // Result gets pushed on the stack. 11710b57cec5SDimitry Andric pushReg(DestReg); 1172fe6060f1SDimitry Andric 1173fe6060f1SDimitry Andric MI.dropDebugNumber(); 11740b57cec5SDimitry Andric } 11750b57cec5SDimitry Andric 11760b57cec5SDimitry Andric /// handleOneArgFP - fst <mem>, ST(0) 11770b57cec5SDimitry Andric /// 11780b57cec5SDimitry Andric void FPS::handleOneArgFP(MachineBasicBlock::iterator &I) { 11790b57cec5SDimitry Andric MachineInstr &MI = *I; 11800b57cec5SDimitry Andric unsigned NumOps = MI.getDesc().getNumOperands(); 11810b57cec5SDimitry Andric assert((NumOps == X86::AddrNumOperands + 1 || NumOps == 1) && 11820b57cec5SDimitry Andric "Can only handle fst* & ftst instructions!"); 11830b57cec5SDimitry Andric 11840b57cec5SDimitry Andric // Is this the last use of the source register? 11850b57cec5SDimitry Andric unsigned Reg = getFPReg(MI.getOperand(NumOps - 1)); 1186*0fca6ea1SDimitry Andric bool KillsSrc = MI.killsRegister(X86::FP0 + Reg, /*TRI=*/nullptr); 11870b57cec5SDimitry Andric 11880b57cec5SDimitry Andric // FISTP64m is strange because there isn't a non-popping versions. 11890b57cec5SDimitry Andric // If we have one _and_ we don't want to pop the operand, duplicate the value 11900b57cec5SDimitry Andric // on the stack instead of moving it. This ensure that popping the value is 11910b57cec5SDimitry Andric // always ok. 11920b57cec5SDimitry Andric // Ditto FISTTP16m, FISTTP32m, FISTTP64m, ST_FpP80m. 11930b57cec5SDimitry Andric // 11940b57cec5SDimitry Andric if (!KillsSrc && (MI.getOpcode() == X86::IST_Fp64m32 || 11950b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp16m32 || 11960b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp32m32 || 11970b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp64m32 || 11980b57cec5SDimitry Andric MI.getOpcode() == X86::IST_Fp64m64 || 11990b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp16m64 || 12000b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp32m64 || 12010b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp64m64 || 12020b57cec5SDimitry Andric MI.getOpcode() == X86::IST_Fp64m80 || 12030b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp16m80 || 12040b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp32m80 || 12050b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_Fp64m80 || 12060b57cec5SDimitry Andric MI.getOpcode() == X86::ST_FpP80m)) { 12070b57cec5SDimitry Andric duplicateToTop(Reg, ScratchFPReg, I); 12080b57cec5SDimitry Andric } else { 12090b57cec5SDimitry Andric moveToTop(Reg, I); // Move to the top of the stack... 12100b57cec5SDimitry Andric } 12110b57cec5SDimitry Andric 12120b57cec5SDimitry Andric // Convert from the pseudo instruction to the concrete instruction. 121381ad6265SDimitry Andric MI.removeOperand(NumOps - 1); // Remove explicit ST(0) operand 12140b57cec5SDimitry Andric MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode()))); 12150b57cec5SDimitry Andric MI.addOperand( 12160b57cec5SDimitry Andric MachineOperand::CreateReg(X86::ST0, /*isDef*/ false, /*isImp*/ true)); 12170b57cec5SDimitry Andric 12180b57cec5SDimitry Andric if (MI.getOpcode() == X86::IST_FP64m || MI.getOpcode() == X86::ISTT_FP16m || 12190b57cec5SDimitry Andric MI.getOpcode() == X86::ISTT_FP32m || MI.getOpcode() == X86::ISTT_FP64m || 12200b57cec5SDimitry Andric MI.getOpcode() == X86::ST_FP80m) { 12210b57cec5SDimitry Andric if (StackTop == 0) 12220b57cec5SDimitry Andric report_fatal_error("Stack empty??"); 12230b57cec5SDimitry Andric --StackTop; 12240b57cec5SDimitry Andric } else if (KillsSrc) { // Last use of operand? 12250b57cec5SDimitry Andric popStackAfter(I); 12260b57cec5SDimitry Andric } 1227fe6060f1SDimitry Andric 1228fe6060f1SDimitry Andric MI.dropDebugNumber(); 12290b57cec5SDimitry Andric } 12300b57cec5SDimitry Andric 12310b57cec5SDimitry Andric 12320b57cec5SDimitry Andric /// handleOneArgFPRW: Handle instructions that read from the top of stack and 12330b57cec5SDimitry Andric /// replace the value with a newly computed value. These instructions may have 12340b57cec5SDimitry Andric /// non-fp operands after their FP operands. 12350b57cec5SDimitry Andric /// 12360b57cec5SDimitry Andric /// Examples: 12370b57cec5SDimitry Andric /// R1 = fchs R2 12380b57cec5SDimitry Andric /// R1 = fadd R2, [mem] 12390b57cec5SDimitry Andric /// 12400b57cec5SDimitry Andric void FPS::handleOneArgFPRW(MachineBasicBlock::iterator &I) { 12410b57cec5SDimitry Andric MachineInstr &MI = *I; 12420b57cec5SDimitry Andric #ifndef NDEBUG 12430b57cec5SDimitry Andric unsigned NumOps = MI.getDesc().getNumOperands(); 12440b57cec5SDimitry Andric assert(NumOps >= 2 && "FPRW instructions must have 2 ops!!"); 12450b57cec5SDimitry Andric #endif 12460b57cec5SDimitry Andric 12470b57cec5SDimitry Andric // Is this the last use of the source register? 12480b57cec5SDimitry Andric unsigned Reg = getFPReg(MI.getOperand(1)); 1249*0fca6ea1SDimitry Andric bool KillsSrc = MI.killsRegister(X86::FP0 + Reg, /*TRI=*/nullptr); 12500b57cec5SDimitry Andric 12510b57cec5SDimitry Andric if (KillsSrc) { 12520b57cec5SDimitry Andric // If this is the last use of the source register, just make sure it's on 12530b57cec5SDimitry Andric // the top of the stack. 12540b57cec5SDimitry Andric moveToTop(Reg, I); 12550b57cec5SDimitry Andric if (StackTop == 0) 12560b57cec5SDimitry Andric report_fatal_error("Stack cannot be empty!"); 12570b57cec5SDimitry Andric --StackTop; 12580b57cec5SDimitry Andric pushReg(getFPReg(MI.getOperand(0))); 12590b57cec5SDimitry Andric } else { 12600b57cec5SDimitry Andric // If this is not the last use of the source register, _copy_ it to the top 12610b57cec5SDimitry Andric // of the stack. 12620b57cec5SDimitry Andric duplicateToTop(Reg, getFPReg(MI.getOperand(0)), I); 12630b57cec5SDimitry Andric } 12640b57cec5SDimitry Andric 12650b57cec5SDimitry Andric // Change from the pseudo instruction to the concrete instruction. 126681ad6265SDimitry Andric MI.removeOperand(1); // Drop the source operand. 126781ad6265SDimitry Andric MI.removeOperand(0); // Drop the destination operand. 12680b57cec5SDimitry Andric MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode()))); 1269fe6060f1SDimitry Andric MI.dropDebugNumber(); 12700b57cec5SDimitry Andric } 12710b57cec5SDimitry Andric 12720b57cec5SDimitry Andric 12730b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 12740b57cec5SDimitry Andric // Define tables of various ways to map pseudo instructions 12750b57cec5SDimitry Andric // 12760b57cec5SDimitry Andric 12770b57cec5SDimitry Andric // ForwardST0Table - Map: A = B op C into: ST(0) = ST(0) op ST(i) 12780b57cec5SDimitry Andric static const TableEntry ForwardST0Table[] = { 12790b57cec5SDimitry Andric { X86::ADD_Fp32 , X86::ADD_FST0r }, 12800b57cec5SDimitry Andric { X86::ADD_Fp64 , X86::ADD_FST0r }, 12810b57cec5SDimitry Andric { X86::ADD_Fp80 , X86::ADD_FST0r }, 12820b57cec5SDimitry Andric { X86::DIV_Fp32 , X86::DIV_FST0r }, 12830b57cec5SDimitry Andric { X86::DIV_Fp64 , X86::DIV_FST0r }, 12840b57cec5SDimitry Andric { X86::DIV_Fp80 , X86::DIV_FST0r }, 12850b57cec5SDimitry Andric { X86::MUL_Fp32 , X86::MUL_FST0r }, 12860b57cec5SDimitry Andric { X86::MUL_Fp64 , X86::MUL_FST0r }, 12870b57cec5SDimitry Andric { X86::MUL_Fp80 , X86::MUL_FST0r }, 12880b57cec5SDimitry Andric { X86::SUB_Fp32 , X86::SUB_FST0r }, 12890b57cec5SDimitry Andric { X86::SUB_Fp64 , X86::SUB_FST0r }, 12900b57cec5SDimitry Andric { X86::SUB_Fp80 , X86::SUB_FST0r }, 12910b57cec5SDimitry Andric }; 12920b57cec5SDimitry Andric 12930b57cec5SDimitry Andric // ReverseST0Table - Map: A = B op C into: ST(0) = ST(i) op ST(0) 12940b57cec5SDimitry Andric static const TableEntry ReverseST0Table[] = { 12950b57cec5SDimitry Andric { X86::ADD_Fp32 , X86::ADD_FST0r }, // commutative 12960b57cec5SDimitry Andric { X86::ADD_Fp64 , X86::ADD_FST0r }, // commutative 12970b57cec5SDimitry Andric { X86::ADD_Fp80 , X86::ADD_FST0r }, // commutative 12980b57cec5SDimitry Andric { X86::DIV_Fp32 , X86::DIVR_FST0r }, 12990b57cec5SDimitry Andric { X86::DIV_Fp64 , X86::DIVR_FST0r }, 13000b57cec5SDimitry Andric { X86::DIV_Fp80 , X86::DIVR_FST0r }, 13010b57cec5SDimitry Andric { X86::MUL_Fp32 , X86::MUL_FST0r }, // commutative 13020b57cec5SDimitry Andric { X86::MUL_Fp64 , X86::MUL_FST0r }, // commutative 13030b57cec5SDimitry Andric { X86::MUL_Fp80 , X86::MUL_FST0r }, // commutative 13040b57cec5SDimitry Andric { X86::SUB_Fp32 , X86::SUBR_FST0r }, 13050b57cec5SDimitry Andric { X86::SUB_Fp64 , X86::SUBR_FST0r }, 13060b57cec5SDimitry Andric { X86::SUB_Fp80 , X86::SUBR_FST0r }, 13070b57cec5SDimitry Andric }; 13080b57cec5SDimitry Andric 13090b57cec5SDimitry Andric // ForwardSTiTable - Map: A = B op C into: ST(i) = ST(0) op ST(i) 13100b57cec5SDimitry Andric static const TableEntry ForwardSTiTable[] = { 13110b57cec5SDimitry Andric { X86::ADD_Fp32 , X86::ADD_FrST0 }, // commutative 13120b57cec5SDimitry Andric { X86::ADD_Fp64 , X86::ADD_FrST0 }, // commutative 13130b57cec5SDimitry Andric { X86::ADD_Fp80 , X86::ADD_FrST0 }, // commutative 13140b57cec5SDimitry Andric { X86::DIV_Fp32 , X86::DIVR_FrST0 }, 13150b57cec5SDimitry Andric { X86::DIV_Fp64 , X86::DIVR_FrST0 }, 13160b57cec5SDimitry Andric { X86::DIV_Fp80 , X86::DIVR_FrST0 }, 13170b57cec5SDimitry Andric { X86::MUL_Fp32 , X86::MUL_FrST0 }, // commutative 13180b57cec5SDimitry Andric { X86::MUL_Fp64 , X86::MUL_FrST0 }, // commutative 13190b57cec5SDimitry Andric { X86::MUL_Fp80 , X86::MUL_FrST0 }, // commutative 13200b57cec5SDimitry Andric { X86::SUB_Fp32 , X86::SUBR_FrST0 }, 13210b57cec5SDimitry Andric { X86::SUB_Fp64 , X86::SUBR_FrST0 }, 13220b57cec5SDimitry Andric { X86::SUB_Fp80 , X86::SUBR_FrST0 }, 13230b57cec5SDimitry Andric }; 13240b57cec5SDimitry Andric 13250b57cec5SDimitry Andric // ReverseSTiTable - Map: A = B op C into: ST(i) = ST(i) op ST(0) 13260b57cec5SDimitry Andric static const TableEntry ReverseSTiTable[] = { 13270b57cec5SDimitry Andric { X86::ADD_Fp32 , X86::ADD_FrST0 }, 13280b57cec5SDimitry Andric { X86::ADD_Fp64 , X86::ADD_FrST0 }, 13290b57cec5SDimitry Andric { X86::ADD_Fp80 , X86::ADD_FrST0 }, 13300b57cec5SDimitry Andric { X86::DIV_Fp32 , X86::DIV_FrST0 }, 13310b57cec5SDimitry Andric { X86::DIV_Fp64 , X86::DIV_FrST0 }, 13320b57cec5SDimitry Andric { X86::DIV_Fp80 , X86::DIV_FrST0 }, 13330b57cec5SDimitry Andric { X86::MUL_Fp32 , X86::MUL_FrST0 }, 13340b57cec5SDimitry Andric { X86::MUL_Fp64 , X86::MUL_FrST0 }, 13350b57cec5SDimitry Andric { X86::MUL_Fp80 , X86::MUL_FrST0 }, 13360b57cec5SDimitry Andric { X86::SUB_Fp32 , X86::SUB_FrST0 }, 13370b57cec5SDimitry Andric { X86::SUB_Fp64 , X86::SUB_FrST0 }, 13380b57cec5SDimitry Andric { X86::SUB_Fp80 , X86::SUB_FrST0 }, 13390b57cec5SDimitry Andric }; 13400b57cec5SDimitry Andric 13410b57cec5SDimitry Andric 13420b57cec5SDimitry Andric /// handleTwoArgFP - Handle instructions like FADD and friends which are virtual 13430b57cec5SDimitry Andric /// instructions which need to be simplified and possibly transformed. 13440b57cec5SDimitry Andric /// 13450b57cec5SDimitry Andric /// Result: ST(0) = fsub ST(0), ST(i) 13460b57cec5SDimitry Andric /// ST(i) = fsub ST(0), ST(i) 13470b57cec5SDimitry Andric /// ST(0) = fsubr ST(0), ST(i) 13480b57cec5SDimitry Andric /// ST(i) = fsubr ST(0), ST(i) 13490b57cec5SDimitry Andric /// 13500b57cec5SDimitry Andric void FPS::handleTwoArgFP(MachineBasicBlock::iterator &I) { 13510b57cec5SDimitry Andric ASSERT_SORTED(ForwardST0Table); ASSERT_SORTED(ReverseST0Table); 13520b57cec5SDimitry Andric ASSERT_SORTED(ForwardSTiTable); ASSERT_SORTED(ReverseSTiTable); 13530b57cec5SDimitry Andric MachineInstr &MI = *I; 13540b57cec5SDimitry Andric 13550b57cec5SDimitry Andric unsigned NumOperands = MI.getDesc().getNumOperands(); 13560b57cec5SDimitry Andric assert(NumOperands == 3 && "Illegal TwoArgFP instruction!"); 13570b57cec5SDimitry Andric unsigned Dest = getFPReg(MI.getOperand(0)); 13580b57cec5SDimitry Andric unsigned Op0 = getFPReg(MI.getOperand(NumOperands - 2)); 13590b57cec5SDimitry Andric unsigned Op1 = getFPReg(MI.getOperand(NumOperands - 1)); 1360*0fca6ea1SDimitry Andric bool KillsOp0 = MI.killsRegister(X86::FP0 + Op0, /*TRI=*/nullptr); 1361*0fca6ea1SDimitry Andric bool KillsOp1 = MI.killsRegister(X86::FP0 + Op1, /*TRI=*/nullptr); 1362fe6060f1SDimitry Andric const DebugLoc &dl = MI.getDebugLoc(); 13630b57cec5SDimitry Andric 13640b57cec5SDimitry Andric unsigned TOS = getStackEntry(0); 13650b57cec5SDimitry Andric 13660b57cec5SDimitry Andric // One of our operands must be on the top of the stack. If neither is yet, we 13670b57cec5SDimitry Andric // need to move one. 13680b57cec5SDimitry Andric if (Op0 != TOS && Op1 != TOS) { // No operand at TOS? 13690b57cec5SDimitry Andric // We can choose to move either operand to the top of the stack. If one of 13700b57cec5SDimitry Andric // the operands is killed by this instruction, we want that one so that we 13710b57cec5SDimitry Andric // can update right on top of the old version. 13720b57cec5SDimitry Andric if (KillsOp0) { 13730b57cec5SDimitry Andric moveToTop(Op0, I); // Move dead operand to TOS. 13740b57cec5SDimitry Andric TOS = Op0; 13750b57cec5SDimitry Andric } else if (KillsOp1) { 13760b57cec5SDimitry Andric moveToTop(Op1, I); 13770b57cec5SDimitry Andric TOS = Op1; 13780b57cec5SDimitry Andric } else { 13790b57cec5SDimitry Andric // All of the operands are live after this instruction executes, so we 13800b57cec5SDimitry Andric // cannot update on top of any operand. Because of this, we must 13810b57cec5SDimitry Andric // duplicate one of the stack elements to the top. It doesn't matter 13820b57cec5SDimitry Andric // which one we pick. 13830b57cec5SDimitry Andric // 13840b57cec5SDimitry Andric duplicateToTop(Op0, Dest, I); 13850b57cec5SDimitry Andric Op0 = TOS = Dest; 13860b57cec5SDimitry Andric KillsOp0 = true; 13870b57cec5SDimitry Andric } 13880b57cec5SDimitry Andric } else if (!KillsOp0 && !KillsOp1) { 13890b57cec5SDimitry Andric // If we DO have one of our operands at the top of the stack, but we don't 13900b57cec5SDimitry Andric // have a dead operand, we must duplicate one of the operands to a new slot 13910b57cec5SDimitry Andric // on the stack. 13920b57cec5SDimitry Andric duplicateToTop(Op0, Dest, I); 13930b57cec5SDimitry Andric Op0 = TOS = Dest; 13940b57cec5SDimitry Andric KillsOp0 = true; 13950b57cec5SDimitry Andric } 13960b57cec5SDimitry Andric 13970b57cec5SDimitry Andric // Now we know that one of our operands is on the top of the stack, and at 13980b57cec5SDimitry Andric // least one of our operands is killed by this instruction. 13990b57cec5SDimitry Andric assert((TOS == Op0 || TOS == Op1) && (KillsOp0 || KillsOp1) && 14000b57cec5SDimitry Andric "Stack conditions not set up right!"); 14010b57cec5SDimitry Andric 14020b57cec5SDimitry Andric // We decide which form to use based on what is on the top of the stack, and 14030b57cec5SDimitry Andric // which operand is killed by this instruction. 14040b57cec5SDimitry Andric ArrayRef<TableEntry> InstTable; 14050b57cec5SDimitry Andric bool isForward = TOS == Op0; 14060b57cec5SDimitry Andric bool updateST0 = (TOS == Op0 && !KillsOp1) || (TOS == Op1 && !KillsOp0); 14070b57cec5SDimitry Andric if (updateST0) { 14080b57cec5SDimitry Andric if (isForward) 14090b57cec5SDimitry Andric InstTable = ForwardST0Table; 14100b57cec5SDimitry Andric else 14110b57cec5SDimitry Andric InstTable = ReverseST0Table; 14120b57cec5SDimitry Andric } else { 14130b57cec5SDimitry Andric if (isForward) 14140b57cec5SDimitry Andric InstTable = ForwardSTiTable; 14150b57cec5SDimitry Andric else 14160b57cec5SDimitry Andric InstTable = ReverseSTiTable; 14170b57cec5SDimitry Andric } 14180b57cec5SDimitry Andric 14190b57cec5SDimitry Andric int Opcode = Lookup(InstTable, MI.getOpcode()); 14200b57cec5SDimitry Andric assert(Opcode != -1 && "Unknown TwoArgFP pseudo instruction!"); 14210b57cec5SDimitry Andric 14220b57cec5SDimitry Andric // NotTOS - The register which is not on the top of stack... 14230b57cec5SDimitry Andric unsigned NotTOS = (TOS == Op0) ? Op1 : Op0; 14240b57cec5SDimitry Andric 14250b57cec5SDimitry Andric // Replace the old instruction with a new instruction 14260b57cec5SDimitry Andric MBB->remove(&*I++); 14270b57cec5SDimitry Andric I = BuildMI(*MBB, I, dl, TII->get(Opcode)).addReg(getSTReg(NotTOS)); 14280b57cec5SDimitry Andric 14295ffd83dbSDimitry Andric if (!MI.mayRaiseFPException()) 14305ffd83dbSDimitry Andric I->setFlag(MachineInstr::MIFlag::NoFPExcept); 14315ffd83dbSDimitry Andric 14320b57cec5SDimitry Andric // If both operands are killed, pop one off of the stack in addition to 14330b57cec5SDimitry Andric // overwriting the other one. 14340b57cec5SDimitry Andric if (KillsOp0 && KillsOp1 && Op0 != Op1) { 14350b57cec5SDimitry Andric assert(!updateST0 && "Should have updated other operand!"); 14360b57cec5SDimitry Andric popStackAfter(I); // Pop the top of stack 14370b57cec5SDimitry Andric } 14380b57cec5SDimitry Andric 14390b57cec5SDimitry Andric // Update stack information so that we know the destination register is now on 14400b57cec5SDimitry Andric // the stack. 14410b57cec5SDimitry Andric unsigned UpdatedSlot = getSlot(updateST0 ? TOS : NotTOS); 14420b57cec5SDimitry Andric assert(UpdatedSlot < StackTop && Dest < 7); 14430b57cec5SDimitry Andric Stack[UpdatedSlot] = Dest; 14440b57cec5SDimitry Andric RegMap[Dest] = UpdatedSlot; 14450eae32dcSDimitry Andric MBB->getParent()->deleteMachineInstr(&MI); // Remove the old instruction 14460b57cec5SDimitry Andric } 14470b57cec5SDimitry Andric 14480b57cec5SDimitry Andric /// handleCompareFP - Handle FUCOM and FUCOMI instructions, which have two FP 14490b57cec5SDimitry Andric /// register arguments and no explicit destinations. 14500b57cec5SDimitry Andric /// 14510b57cec5SDimitry Andric void FPS::handleCompareFP(MachineBasicBlock::iterator &I) { 14520b57cec5SDimitry Andric MachineInstr &MI = *I; 14530b57cec5SDimitry Andric 14540b57cec5SDimitry Andric unsigned NumOperands = MI.getDesc().getNumOperands(); 14550b57cec5SDimitry Andric assert(NumOperands == 2 && "Illegal FUCOM* instruction!"); 14560b57cec5SDimitry Andric unsigned Op0 = getFPReg(MI.getOperand(NumOperands - 2)); 14570b57cec5SDimitry Andric unsigned Op1 = getFPReg(MI.getOperand(NumOperands - 1)); 1458*0fca6ea1SDimitry Andric bool KillsOp0 = MI.killsRegister(X86::FP0 + Op0, /*TRI=*/nullptr); 1459*0fca6ea1SDimitry Andric bool KillsOp1 = MI.killsRegister(X86::FP0 + Op1, /*TRI=*/nullptr); 14600b57cec5SDimitry Andric 14610b57cec5SDimitry Andric // Make sure the first operand is on the top of stack, the other one can be 14620b57cec5SDimitry Andric // anywhere. 14630b57cec5SDimitry Andric moveToTop(Op0, I); 14640b57cec5SDimitry Andric 14650b57cec5SDimitry Andric // Change from the pseudo instruction to the concrete instruction. 14660b57cec5SDimitry Andric MI.getOperand(0).setReg(getSTReg(Op1)); 146781ad6265SDimitry Andric MI.removeOperand(1); 14680b57cec5SDimitry Andric MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode()))); 1469fe6060f1SDimitry Andric MI.dropDebugNumber(); 14700b57cec5SDimitry Andric 14710b57cec5SDimitry Andric // If any of the operands are killed by this instruction, free them. 14720b57cec5SDimitry Andric if (KillsOp0) freeStackSlotAfter(I, Op0); 14730b57cec5SDimitry Andric if (KillsOp1 && Op0 != Op1) freeStackSlotAfter(I, Op1); 14740b57cec5SDimitry Andric } 14750b57cec5SDimitry Andric 14760b57cec5SDimitry Andric /// handleCondMovFP - Handle two address conditional move instructions. These 14770b57cec5SDimitry Andric /// instructions move a st(i) register to st(0) iff a condition is true. These 14780b57cec5SDimitry Andric /// instructions require that the first operand is at the top of the stack, but 14790b57cec5SDimitry Andric /// otherwise don't modify the stack at all. 14800b57cec5SDimitry Andric void FPS::handleCondMovFP(MachineBasicBlock::iterator &I) { 14810b57cec5SDimitry Andric MachineInstr &MI = *I; 14820b57cec5SDimitry Andric 14830b57cec5SDimitry Andric unsigned Op0 = getFPReg(MI.getOperand(0)); 14840b57cec5SDimitry Andric unsigned Op1 = getFPReg(MI.getOperand(2)); 1485*0fca6ea1SDimitry Andric bool KillsOp1 = MI.killsRegister(X86::FP0 + Op1, /*TRI=*/nullptr); 14860b57cec5SDimitry Andric 14870b57cec5SDimitry Andric // The first operand *must* be on the top of the stack. 14880b57cec5SDimitry Andric moveToTop(Op0, I); 14890b57cec5SDimitry Andric 14900b57cec5SDimitry Andric // Change the second operand to the stack register that the operand is in. 14910b57cec5SDimitry Andric // Change from the pseudo instruction to the concrete instruction. 149281ad6265SDimitry Andric MI.removeOperand(0); 149381ad6265SDimitry Andric MI.removeOperand(1); 14940b57cec5SDimitry Andric MI.getOperand(0).setReg(getSTReg(Op1)); 14950b57cec5SDimitry Andric MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode()))); 1496fe6060f1SDimitry Andric MI.dropDebugNumber(); 14970b57cec5SDimitry Andric 14980b57cec5SDimitry Andric // If we kill the second operand, make sure to pop it from the stack. 14990b57cec5SDimitry Andric if (Op0 != Op1 && KillsOp1) { 15000b57cec5SDimitry Andric // Get this value off of the register stack. 15010b57cec5SDimitry Andric freeStackSlotAfter(I, Op1); 15020b57cec5SDimitry Andric } 15030b57cec5SDimitry Andric } 15040b57cec5SDimitry Andric 15050b57cec5SDimitry Andric 15060b57cec5SDimitry Andric /// handleSpecialFP - Handle special instructions which behave unlike other 15070b57cec5SDimitry Andric /// floating point instructions. This is primarily intended for use by pseudo 15080b57cec5SDimitry Andric /// instructions. 15090b57cec5SDimitry Andric /// 15100b57cec5SDimitry Andric void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) { 15110b57cec5SDimitry Andric MachineInstr &MI = *Inst; 15120b57cec5SDimitry Andric 15130b57cec5SDimitry Andric if (MI.isCall()) { 15140b57cec5SDimitry Andric handleCall(Inst); 15150b57cec5SDimitry Andric return; 15160b57cec5SDimitry Andric } 15170b57cec5SDimitry Andric 15180b57cec5SDimitry Andric if (MI.isReturn()) { 15190b57cec5SDimitry Andric handleReturn(Inst); 15200b57cec5SDimitry Andric return; 15210b57cec5SDimitry Andric } 15220b57cec5SDimitry Andric 15230b57cec5SDimitry Andric switch (MI.getOpcode()) { 15240b57cec5SDimitry Andric default: llvm_unreachable("Unknown SpecialFP instruction!"); 15250b57cec5SDimitry Andric case TargetOpcode::COPY: { 15260b57cec5SDimitry Andric // We handle three kinds of copies: FP <- FP, FP <- ST, and ST <- FP. 15270b57cec5SDimitry Andric const MachineOperand &MO1 = MI.getOperand(1); 15280b57cec5SDimitry Andric const MachineOperand &MO0 = MI.getOperand(0); 1529*0fca6ea1SDimitry Andric bool KillsSrc = MI.killsRegister(MO1.getReg(), /*TRI=*/nullptr); 15300b57cec5SDimitry Andric 15310b57cec5SDimitry Andric // FP <- FP copy. 15320b57cec5SDimitry Andric unsigned DstFP = getFPReg(MO0); 15330b57cec5SDimitry Andric unsigned SrcFP = getFPReg(MO1); 15340b57cec5SDimitry Andric assert(isLive(SrcFP) && "Cannot copy dead register"); 15350b57cec5SDimitry Andric if (KillsSrc) { 15360b57cec5SDimitry Andric // If the input operand is killed, we can just change the owner of the 15370b57cec5SDimitry Andric // incoming stack slot into the result. 15380b57cec5SDimitry Andric unsigned Slot = getSlot(SrcFP); 15390b57cec5SDimitry Andric Stack[Slot] = DstFP; 15400b57cec5SDimitry Andric RegMap[DstFP] = Slot; 15410b57cec5SDimitry Andric } else { 15420b57cec5SDimitry Andric // For COPY we just duplicate the specified value to a new stack slot. 15430b57cec5SDimitry Andric // This could be made better, but would require substantial changes. 15440b57cec5SDimitry Andric duplicateToTop(SrcFP, DstFP, Inst); 15450b57cec5SDimitry Andric } 15460b57cec5SDimitry Andric break; 15470b57cec5SDimitry Andric } 15480b57cec5SDimitry Andric 15490b57cec5SDimitry Andric case TargetOpcode::IMPLICIT_DEF: { 15500b57cec5SDimitry Andric // All FP registers must be explicitly defined, so load a 0 instead. 15510b57cec5SDimitry Andric unsigned Reg = MI.getOperand(0).getReg() - X86::FP0; 15520b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Emitting LD_F0 for implicit FP" << Reg << '\n'); 15530b57cec5SDimitry Andric BuildMI(*MBB, Inst, MI.getDebugLoc(), TII->get(X86::LD_F0)); 15540b57cec5SDimitry Andric pushReg(Reg); 15550b57cec5SDimitry Andric break; 15560b57cec5SDimitry Andric } 15570b57cec5SDimitry Andric 15580b57cec5SDimitry Andric case TargetOpcode::INLINEASM: 15590b57cec5SDimitry Andric case TargetOpcode::INLINEASM_BR: { 15600b57cec5SDimitry Andric // The inline asm MachineInstr currently only *uses* FP registers for the 15610b57cec5SDimitry Andric // 'f' constraint. These should be turned into the current ST(x) register 15620b57cec5SDimitry Andric // in the machine instr. 15630b57cec5SDimitry Andric // 15640b57cec5SDimitry Andric // There are special rules for x87 inline assembly. The compiler must know 15650b57cec5SDimitry Andric // exactly how many registers are popped and pushed implicitly by the asm. 15660b57cec5SDimitry Andric // Otherwise it is not possible to restore the stack state after the inline 15670b57cec5SDimitry Andric // asm. 15680b57cec5SDimitry Andric // 15690b57cec5SDimitry Andric // There are 3 kinds of input operands: 15700b57cec5SDimitry Andric // 15710b57cec5SDimitry Andric // 1. Popped inputs. These must appear at the stack top in ST0-STn. A 15720b57cec5SDimitry Andric // popped input operand must be in a fixed stack slot, and it is either 15730b57cec5SDimitry Andric // tied to an output operand, or in the clobber list. The MI has ST use 15740b57cec5SDimitry Andric // and def operands for these inputs. 15750b57cec5SDimitry Andric // 15760b57cec5SDimitry Andric // 2. Fixed inputs. These inputs appear in fixed stack slots, but are 15770b57cec5SDimitry Andric // preserved by the inline asm. The fixed stack slots must be STn-STm 15780b57cec5SDimitry Andric // following the popped inputs. A fixed input operand cannot be tied to 15790b57cec5SDimitry Andric // an output or appear in the clobber list. The MI has ST use operands 15800b57cec5SDimitry Andric // and no defs for these inputs. 15810b57cec5SDimitry Andric // 15820b57cec5SDimitry Andric // 3. Preserved inputs. These inputs use the "f" constraint which is 15830b57cec5SDimitry Andric // represented as an FP register. The inline asm won't change these 15840b57cec5SDimitry Andric // stack slots. 15850b57cec5SDimitry Andric // 15860b57cec5SDimitry Andric // Outputs must be in ST registers, FP outputs are not allowed. Clobbered 15870b57cec5SDimitry Andric // registers do not count as output operands. The inline asm changes the 15880b57cec5SDimitry Andric // stack as if it popped all the popped inputs and then pushed all the 15890b57cec5SDimitry Andric // output operands. 15900b57cec5SDimitry Andric 15910b57cec5SDimitry Andric // Scan the assembly for ST registers used, defined and clobbered. We can 15920b57cec5SDimitry Andric // only tell clobbers from defs by looking at the asm descriptor. 1593fe6060f1SDimitry Andric unsigned STUses = 0, STDefs = 0, STClobbers = 0; 15940b57cec5SDimitry Andric unsigned NumOps = 0; 15950b57cec5SDimitry Andric SmallSet<unsigned, 1> FRegIdx; 15960b57cec5SDimitry Andric unsigned RCID; 15970b57cec5SDimitry Andric 15980b57cec5SDimitry Andric for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI.getNumOperands(); 15990b57cec5SDimitry Andric i != e && MI.getOperand(i).isImm(); i += 1 + NumOps) { 16000b57cec5SDimitry Andric unsigned Flags = MI.getOperand(i).getImm(); 16015f757f3fSDimitry Andric const InlineAsm::Flag F(Flags); 16020b57cec5SDimitry Andric 16035f757f3fSDimitry Andric NumOps = F.getNumOperandRegisters(); 16040b57cec5SDimitry Andric if (NumOps != 1) 16050b57cec5SDimitry Andric continue; 16060b57cec5SDimitry Andric const MachineOperand &MO = MI.getOperand(i + 1); 16070b57cec5SDimitry Andric if (!MO.isReg()) 16080b57cec5SDimitry Andric continue; 16090b57cec5SDimitry Andric unsigned STReg = MO.getReg() - X86::FP0; 16100b57cec5SDimitry Andric if (STReg >= 8) 16110b57cec5SDimitry Andric continue; 16120b57cec5SDimitry Andric 16130b57cec5SDimitry Andric // If the flag has a register class constraint, this must be an operand 16140b57cec5SDimitry Andric // with constraint "f". Record its index and continue. 16155f757f3fSDimitry Andric if (F.hasRegClassConstraint(RCID)) { 16160b57cec5SDimitry Andric FRegIdx.insert(i + 1); 16170b57cec5SDimitry Andric continue; 16180b57cec5SDimitry Andric } 16190b57cec5SDimitry Andric 16205f757f3fSDimitry Andric switch (F.getKind()) { 16215f757f3fSDimitry Andric case InlineAsm::Kind::RegUse: 16220b57cec5SDimitry Andric STUses |= (1u << STReg); 16230b57cec5SDimitry Andric break; 16245f757f3fSDimitry Andric case InlineAsm::Kind::RegDef: 16255f757f3fSDimitry Andric case InlineAsm::Kind::RegDefEarlyClobber: 16260b57cec5SDimitry Andric STDefs |= (1u << STReg); 16270b57cec5SDimitry Andric break; 16285f757f3fSDimitry Andric case InlineAsm::Kind::Clobber: 16290b57cec5SDimitry Andric STClobbers |= (1u << STReg); 16300b57cec5SDimitry Andric break; 16310b57cec5SDimitry Andric default: 16320b57cec5SDimitry Andric break; 16330b57cec5SDimitry Andric } 16340b57cec5SDimitry Andric } 16350b57cec5SDimitry Andric 16360b57cec5SDimitry Andric if (STUses && !isMask_32(STUses)) 16370b57cec5SDimitry Andric MI.emitError("fixed input regs must be last on the x87 stack"); 163806c3fb27SDimitry Andric unsigned NumSTUses = llvm::countr_one(STUses); 16390b57cec5SDimitry Andric 16400b57cec5SDimitry Andric // Defs must be contiguous from the stack top. ST0-STn. 16410b57cec5SDimitry Andric if (STDefs && !isMask_32(STDefs)) { 16420b57cec5SDimitry Andric MI.emitError("output regs must be last on the x87 stack"); 16430b57cec5SDimitry Andric STDefs = NextPowerOf2(STDefs) - 1; 16440b57cec5SDimitry Andric } 164506c3fb27SDimitry Andric unsigned NumSTDefs = llvm::countr_one(STDefs); 16460b57cec5SDimitry Andric 16470b57cec5SDimitry Andric // So must the clobbered stack slots. ST0-STm, m >= n. 16480b57cec5SDimitry Andric if (STClobbers && !isMask_32(STDefs | STClobbers)) 16490b57cec5SDimitry Andric MI.emitError("clobbers must be last on the x87 stack"); 16500b57cec5SDimitry Andric 16510b57cec5SDimitry Andric // Popped inputs are the ones that are also clobbered or defined. 16520b57cec5SDimitry Andric unsigned STPopped = STUses & (STDefs | STClobbers); 16530b57cec5SDimitry Andric if (STPopped && !isMask_32(STPopped)) 16540b57cec5SDimitry Andric MI.emitError("implicitly popped regs must be last on the x87 stack"); 165506c3fb27SDimitry Andric unsigned NumSTPopped = llvm::countr_one(STPopped); 16560b57cec5SDimitry Andric 16570b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Asm uses " << NumSTUses << " fixed regs, pops " 16580b57cec5SDimitry Andric << NumSTPopped << ", and defines " << NumSTDefs 16590b57cec5SDimitry Andric << " regs.\n"); 16600b57cec5SDimitry Andric 16610b57cec5SDimitry Andric #ifndef NDEBUG 16620b57cec5SDimitry Andric // If any input operand uses constraint "f", all output register 16630b57cec5SDimitry Andric // constraints must be early-clobber defs. 16640b57cec5SDimitry Andric for (unsigned I = 0, E = MI.getNumOperands(); I < E; ++I) 16650b57cec5SDimitry Andric if (FRegIdx.count(I)) { 16660b57cec5SDimitry Andric assert((1 << getFPReg(MI.getOperand(I)) & STDefs) == 0 && 16670b57cec5SDimitry Andric "Operands with constraint \"f\" cannot overlap with defs"); 16680b57cec5SDimitry Andric } 16690b57cec5SDimitry Andric #endif 16700b57cec5SDimitry Andric 16710b57cec5SDimitry Andric // Collect all FP registers (register operands with constraints "t", "u", 16720b57cec5SDimitry Andric // and "f") to kill afer the instruction. 16730b57cec5SDimitry Andric unsigned FPKills = ((1u << NumFPRegs) - 1) & ~0xff; 16744824e7fdSDimitry Andric for (const MachineOperand &Op : MI.operands()) { 16750b57cec5SDimitry Andric if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6) 16760b57cec5SDimitry Andric continue; 16770b57cec5SDimitry Andric unsigned FPReg = getFPReg(Op); 16780b57cec5SDimitry Andric 16790b57cec5SDimitry Andric // If we kill this operand, make sure to pop it from the stack after the 16800b57cec5SDimitry Andric // asm. We just remember it for now, and pop them all off at the end in 16810b57cec5SDimitry Andric // a batch. 16820b57cec5SDimitry Andric if (Op.isUse() && Op.isKill()) 16830b57cec5SDimitry Andric FPKills |= 1U << FPReg; 16840b57cec5SDimitry Andric } 16850b57cec5SDimitry Andric 16860b57cec5SDimitry Andric // Do not include registers that are implicitly popped by defs/clobbers. 16870b57cec5SDimitry Andric FPKills &= ~(STDefs | STClobbers); 16880b57cec5SDimitry Andric 16890b57cec5SDimitry Andric // Now we can rearrange the live registers to match what was requested. 16900b57cec5SDimitry Andric unsigned char STUsesArray[8]; 16910b57cec5SDimitry Andric 16920b57cec5SDimitry Andric for (unsigned I = 0; I < NumSTUses; ++I) 16930b57cec5SDimitry Andric STUsesArray[I] = I; 16940b57cec5SDimitry Andric 16950b57cec5SDimitry Andric shuffleStackTop(STUsesArray, NumSTUses, Inst); 16960b57cec5SDimitry Andric LLVM_DEBUG({ 16970b57cec5SDimitry Andric dbgs() << "Before asm: "; 16980b57cec5SDimitry Andric dumpStack(); 16990b57cec5SDimitry Andric }); 17000b57cec5SDimitry Andric 17010b57cec5SDimitry Andric // With the stack layout fixed, rewrite the FP registers. 17020b57cec5SDimitry Andric for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 17030b57cec5SDimitry Andric MachineOperand &Op = MI.getOperand(i); 17040b57cec5SDimitry Andric if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6) 17050b57cec5SDimitry Andric continue; 17060b57cec5SDimitry Andric 17070b57cec5SDimitry Andric unsigned FPReg = getFPReg(Op); 17080b57cec5SDimitry Andric 17090b57cec5SDimitry Andric if (FRegIdx.count(i)) 17100b57cec5SDimitry Andric // Operand with constraint "f". 17110b57cec5SDimitry Andric Op.setReg(getSTReg(FPReg)); 17120b57cec5SDimitry Andric else 17130b57cec5SDimitry Andric // Operand with a single register class constraint ("t" or "u"). 17140b57cec5SDimitry Andric Op.setReg(X86::ST0 + FPReg); 17150b57cec5SDimitry Andric } 17160b57cec5SDimitry Andric 17170b57cec5SDimitry Andric // Simulate the inline asm popping its inputs and pushing its outputs. 17180b57cec5SDimitry Andric StackTop -= NumSTPopped; 17190b57cec5SDimitry Andric 17200b57cec5SDimitry Andric for (unsigned i = 0; i < NumSTDefs; ++i) 17210b57cec5SDimitry Andric pushReg(NumSTDefs - i - 1); 17220b57cec5SDimitry Andric 17230b57cec5SDimitry Andric // If this asm kills any FP registers (is the last use of them) we must 17240b57cec5SDimitry Andric // explicitly emit pop instructions for them. Do this now after the asm has 17250b57cec5SDimitry Andric // executed so that the ST(x) numbers are not off (which would happen if we 17260b57cec5SDimitry Andric // did this inline with operand rewriting). 17270b57cec5SDimitry Andric // 17280b57cec5SDimitry Andric // Note: this might be a non-optimal pop sequence. We might be able to do 17290b57cec5SDimitry Andric // better by trying to pop in stack order or something. 17300b57cec5SDimitry Andric while (FPKills) { 173106c3fb27SDimitry Andric unsigned FPReg = llvm::countr_zero(FPKills); 17320b57cec5SDimitry Andric if (isLive(FPReg)) 17330b57cec5SDimitry Andric freeStackSlotAfter(Inst, FPReg); 17340b57cec5SDimitry Andric FPKills &= ~(1U << FPReg); 17350b57cec5SDimitry Andric } 17360b57cec5SDimitry Andric 17370b57cec5SDimitry Andric // Don't delete the inline asm! 17380b57cec5SDimitry Andric return; 17390b57cec5SDimitry Andric } 17400b57cec5SDimitry Andric } 17410b57cec5SDimitry Andric 17420b57cec5SDimitry Andric Inst = MBB->erase(Inst); // Remove the pseudo instruction 17430b57cec5SDimitry Andric 17440b57cec5SDimitry Andric // We want to leave I pointing to the previous instruction, but what if we 17450b57cec5SDimitry Andric // just erased the first instruction? 17460b57cec5SDimitry Andric if (Inst == MBB->begin()) { 17470b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Inserting dummy KILL\n"); 17480b57cec5SDimitry Andric Inst = BuildMI(*MBB, Inst, DebugLoc(), TII->get(TargetOpcode::KILL)); 17490b57cec5SDimitry Andric } else 17500b57cec5SDimitry Andric --Inst; 17510b57cec5SDimitry Andric } 17520b57cec5SDimitry Andric 17530b57cec5SDimitry Andric void FPS::setKillFlags(MachineBasicBlock &MBB) const { 17540b57cec5SDimitry Andric const TargetRegisterInfo &TRI = 17550b57cec5SDimitry Andric *MBB.getParent()->getSubtarget().getRegisterInfo(); 1756*0fca6ea1SDimitry Andric LiveRegUnits LPR(TRI); 17570b57cec5SDimitry Andric 17580b57cec5SDimitry Andric LPR.addLiveOuts(MBB); 17590b57cec5SDimitry Andric 1760349cc55cSDimitry Andric for (MachineInstr &MI : llvm::reverse(MBB)) { 1761349cc55cSDimitry Andric if (MI.isDebugInstr()) 17620b57cec5SDimitry Andric continue; 17630b57cec5SDimitry Andric 17640b57cec5SDimitry Andric std::bitset<8> Defs; 17650b57cec5SDimitry Andric SmallVector<MachineOperand *, 2> Uses; 17660b57cec5SDimitry Andric 1767349cc55cSDimitry Andric for (auto &MO : MI.operands()) { 17680b57cec5SDimitry Andric if (!MO.isReg()) 17690b57cec5SDimitry Andric continue; 17700b57cec5SDimitry Andric 17710b57cec5SDimitry Andric unsigned Reg = MO.getReg() - X86::FP0; 17720b57cec5SDimitry Andric 17730b57cec5SDimitry Andric if (Reg >= 8) 17740b57cec5SDimitry Andric continue; 17750b57cec5SDimitry Andric 17760b57cec5SDimitry Andric if (MO.isDef()) { 17770b57cec5SDimitry Andric Defs.set(Reg); 1778*0fca6ea1SDimitry Andric if (LPR.available(MO.getReg())) 17790b57cec5SDimitry Andric MO.setIsDead(); 17800b57cec5SDimitry Andric } else 17810b57cec5SDimitry Andric Uses.push_back(&MO); 17820b57cec5SDimitry Andric } 17830b57cec5SDimitry Andric 17840b57cec5SDimitry Andric for (auto *MO : Uses) 1785*0fca6ea1SDimitry Andric if (Defs.test(getFPReg(*MO)) || LPR.available(MO->getReg())) 17860b57cec5SDimitry Andric MO->setIsKill(); 17870b57cec5SDimitry Andric 17880b57cec5SDimitry Andric LPR.stepBackward(MI); 17890b57cec5SDimitry Andric } 17900b57cec5SDimitry Andric } 1791