xref: /freebsd-src/contrib/llvm-project/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
10b57cec5SDimitry Andric //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file contains a pass that expands pseudo instructions into target
100b57cec5SDimitry Andric // instructions to allow proper scheduling, if-conversion, and other late
110b57cec5SDimitry Andric // optimizations. This pass should be run after register allocation but before
120b57cec5SDimitry Andric // the post-regalloc scheduling pass.
130b57cec5SDimitry Andric //
140b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
150b57cec5SDimitry Andric 
160b57cec5SDimitry Andric #include "ARM.h"
170b57cec5SDimitry Andric #include "ARMBaseInstrInfo.h"
180b57cec5SDimitry Andric #include "ARMBaseRegisterInfo.h"
190b57cec5SDimitry Andric #include "ARMConstantPoolValue.h"
200b57cec5SDimitry Andric #include "ARMMachineFunctionInfo.h"
210b57cec5SDimitry Andric #include "ARMSubtarget.h"
220b57cec5SDimitry Andric #include "MCTargetDesc/ARMAddressingModes.h"
230b57cec5SDimitry Andric #include "llvm/CodeGen/LivePhysRegs.h"
240b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFrameInfo.h"
250b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunctionPass.h"
2606c3fb27SDimitry Andric #include "llvm/CodeGen/MachineJumpTableInfo.h"
2781ad6265SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
280b57cec5SDimitry Andric #include "llvm/Support/Debug.h"
290b57cec5SDimitry Andric 
300b57cec5SDimitry Andric using namespace llvm;
310b57cec5SDimitry Andric 
320b57cec5SDimitry Andric #define DEBUG_TYPE "arm-pseudo"
330b57cec5SDimitry Andric 
340b57cec5SDimitry Andric static cl::opt<bool>
350b57cec5SDimitry Andric VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
360b57cec5SDimitry Andric                 cl::desc("Verify machine code after expanding ARM pseudos"));
370b57cec5SDimitry Andric 
380b57cec5SDimitry Andric #define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
390b57cec5SDimitry Andric 
400b57cec5SDimitry Andric namespace {
410b57cec5SDimitry Andric   class ARMExpandPseudo : public MachineFunctionPass {
420b57cec5SDimitry Andric   public:
430b57cec5SDimitry Andric     static char ID;
440b57cec5SDimitry Andric     ARMExpandPseudo() : MachineFunctionPass(ID) {}
450b57cec5SDimitry Andric 
460b57cec5SDimitry Andric     const ARMBaseInstrInfo *TII;
470b57cec5SDimitry Andric     const TargetRegisterInfo *TRI;
480b57cec5SDimitry Andric     const ARMSubtarget *STI;
490b57cec5SDimitry Andric     ARMFunctionInfo *AFI;
500b57cec5SDimitry Andric 
510b57cec5SDimitry Andric     bool runOnMachineFunction(MachineFunction &Fn) override;
520b57cec5SDimitry Andric 
530b57cec5SDimitry Andric     MachineFunctionProperties getRequiredProperties() const override {
540b57cec5SDimitry Andric       return MachineFunctionProperties().set(
550b57cec5SDimitry Andric           MachineFunctionProperties::Property::NoVRegs);
560b57cec5SDimitry Andric     }
570b57cec5SDimitry Andric 
580b57cec5SDimitry Andric     StringRef getPassName() const override {
590b57cec5SDimitry Andric       return ARM_EXPAND_PSEUDO_NAME;
600b57cec5SDimitry Andric     }
610b57cec5SDimitry Andric 
620b57cec5SDimitry Andric   private:
630b57cec5SDimitry Andric     bool ExpandMI(MachineBasicBlock &MBB,
640b57cec5SDimitry Andric                   MachineBasicBlock::iterator MBBI,
650b57cec5SDimitry Andric                   MachineBasicBlock::iterator &NextMBBI);
660b57cec5SDimitry Andric     bool ExpandMBB(MachineBasicBlock &MBB);
670b57cec5SDimitry Andric     void ExpandVLD(MachineBasicBlock::iterator &MBBI);
680b57cec5SDimitry Andric     void ExpandVST(MachineBasicBlock::iterator &MBBI);
690b57cec5SDimitry Andric     void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
700b57cec5SDimitry Andric     void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
710b57cec5SDimitry Andric                     unsigned Opc, bool IsExt);
72349cc55cSDimitry Andric     void ExpandMQQPRLoadStore(MachineBasicBlock::iterator &MBBI);
7306c3fb27SDimitry Andric     void ExpandTMOV32BitImm(MachineBasicBlock &MBB,
7406c3fb27SDimitry Andric                             MachineBasicBlock::iterator &MBBI);
750b57cec5SDimitry Andric     void ExpandMOV32BitImm(MachineBasicBlock &MBB,
760b57cec5SDimitry Andric                            MachineBasicBlock::iterator &MBBI);
775ffd83dbSDimitry Andric     void CMSEClearGPRegs(MachineBasicBlock &MBB,
785ffd83dbSDimitry Andric                          MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
795ffd83dbSDimitry Andric                          const SmallVectorImpl<unsigned> &ClearRegs,
805ffd83dbSDimitry Andric                          unsigned ClobberReg);
815ffd83dbSDimitry Andric     MachineBasicBlock &CMSEClearFPRegs(MachineBasicBlock &MBB,
825ffd83dbSDimitry Andric                                        MachineBasicBlock::iterator MBBI);
835ffd83dbSDimitry Andric     MachineBasicBlock &CMSEClearFPRegsV8(MachineBasicBlock &MBB,
845ffd83dbSDimitry Andric                                          MachineBasicBlock::iterator MBBI,
855ffd83dbSDimitry Andric                                          const BitVector &ClearRegs);
865ffd83dbSDimitry Andric     MachineBasicBlock &CMSEClearFPRegsV81(MachineBasicBlock &MBB,
875ffd83dbSDimitry Andric                                           MachineBasicBlock::iterator MBBI,
885ffd83dbSDimitry Andric                                           const BitVector &ClearRegs);
895ffd83dbSDimitry Andric     void CMSESaveClearFPRegs(MachineBasicBlock &MBB,
905ffd83dbSDimitry Andric                              MachineBasicBlock::iterator MBBI, DebugLoc &DL,
915ffd83dbSDimitry Andric                              const LivePhysRegs &LiveRegs,
925ffd83dbSDimitry Andric                              SmallVectorImpl<unsigned> &AvailableRegs);
935ffd83dbSDimitry Andric     void CMSESaveClearFPRegsV8(MachineBasicBlock &MBB,
945ffd83dbSDimitry Andric                                MachineBasicBlock::iterator MBBI, DebugLoc &DL,
955ffd83dbSDimitry Andric                                const LivePhysRegs &LiveRegs,
965ffd83dbSDimitry Andric                                SmallVectorImpl<unsigned> &ScratchRegs);
975ffd83dbSDimitry Andric     void CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
985ffd83dbSDimitry Andric                                 MachineBasicBlock::iterator MBBI, DebugLoc &DL,
995ffd83dbSDimitry Andric                                 const LivePhysRegs &LiveRegs);
1005ffd83dbSDimitry Andric     void CMSERestoreFPRegs(MachineBasicBlock &MBB,
1015ffd83dbSDimitry Andric                            MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1025ffd83dbSDimitry Andric                            SmallVectorImpl<unsigned> &AvailableRegs);
1035ffd83dbSDimitry Andric     void CMSERestoreFPRegsV8(MachineBasicBlock &MBB,
1045ffd83dbSDimitry Andric                              MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1055ffd83dbSDimitry Andric                              SmallVectorImpl<unsigned> &AvailableRegs);
1065ffd83dbSDimitry Andric     void CMSERestoreFPRegsV81(MachineBasicBlock &MBB,
1075ffd83dbSDimitry Andric                               MachineBasicBlock::iterator MBBI, DebugLoc &DL,
1085ffd83dbSDimitry Andric                               SmallVectorImpl<unsigned> &AvailableRegs);
1090b57cec5SDimitry Andric     bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
1100b57cec5SDimitry Andric                         MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
1110b57cec5SDimitry Andric                         unsigned StrexOp, unsigned UxtOp,
1120b57cec5SDimitry Andric                         MachineBasicBlock::iterator &NextMBBI);
1130b57cec5SDimitry Andric 
1140b57cec5SDimitry Andric     bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
1150b57cec5SDimitry Andric                            MachineBasicBlock::iterator MBBI,
1160b57cec5SDimitry Andric                            MachineBasicBlock::iterator &NextMBBI);
1170b57cec5SDimitry Andric   };
1180b57cec5SDimitry Andric   char ARMExpandPseudo::ID = 0;
1190b57cec5SDimitry Andric }
1200b57cec5SDimitry Andric 
1210b57cec5SDimitry Andric INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
1220b57cec5SDimitry Andric                 false)
1230b57cec5SDimitry Andric 
1240b57cec5SDimitry Andric namespace {
1250b57cec5SDimitry Andric   // Constants for register spacing in NEON load/store instructions.
1260b57cec5SDimitry Andric   // For quad-register load-lane and store-lane pseudo instructors, the
1270b57cec5SDimitry Andric   // spacing is initially assumed to be EvenDblSpc, and that is changed to
1280b57cec5SDimitry Andric   // OddDblSpc depending on the lane number operand.
1290b57cec5SDimitry Andric   enum NEONRegSpacing {
1300b57cec5SDimitry Andric     SingleSpc,
1310b57cec5SDimitry Andric     SingleLowSpc ,  // Single spacing, low registers, three and four vectors.
1320b57cec5SDimitry Andric     SingleHighQSpc, // Single spacing, high registers, four vectors.
1330b57cec5SDimitry Andric     SingleHighTSpc, // Single spacing, high registers, three vectors.
1340b57cec5SDimitry Andric     EvenDblSpc,
1350b57cec5SDimitry Andric     OddDblSpc
1360b57cec5SDimitry Andric   };
1370b57cec5SDimitry Andric 
1380b57cec5SDimitry Andric   // Entries for NEON load/store information table.  The table is sorted by
1390b57cec5SDimitry Andric   // PseudoOpc for fast binary-search lookups.
1400b57cec5SDimitry Andric   struct NEONLdStTableEntry {
1410b57cec5SDimitry Andric     uint16_t PseudoOpc;
1420b57cec5SDimitry Andric     uint16_t RealOpc;
1430b57cec5SDimitry Andric     bool IsLoad;
1440b57cec5SDimitry Andric     bool isUpdating;
1450b57cec5SDimitry Andric     bool hasWritebackOperand;
1460b57cec5SDimitry Andric     uint8_t RegSpacing; // One of type NEONRegSpacing
1470b57cec5SDimitry Andric     uint8_t NumRegs; // D registers loaded or stored
1480b57cec5SDimitry Andric     uint8_t RegElts; // elements per D register; used for lane ops
1490b57cec5SDimitry Andric     // FIXME: Temporary flag to denote whether the real instruction takes
1500b57cec5SDimitry Andric     // a single register (like the encoding) or all of the registers in
1510b57cec5SDimitry Andric     // the list (like the asm syntax and the isel DAG). When all definitions
1520b57cec5SDimitry Andric     // are converted to take only the single encoded register, this will
1530b57cec5SDimitry Andric     // go away.
1540b57cec5SDimitry Andric     bool copyAllListRegs;
1550b57cec5SDimitry Andric 
1560b57cec5SDimitry Andric     // Comparison methods for binary search of the table.
1570b57cec5SDimitry Andric     bool operator<(const NEONLdStTableEntry &TE) const {
1580b57cec5SDimitry Andric       return PseudoOpc < TE.PseudoOpc;
1590b57cec5SDimitry Andric     }
1600b57cec5SDimitry Andric     friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
1610b57cec5SDimitry Andric       return TE.PseudoOpc < PseudoOpc;
1620b57cec5SDimitry Andric     }
1630b57cec5SDimitry Andric     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
1640b57cec5SDimitry Andric                                                 const NEONLdStTableEntry &TE) {
1650b57cec5SDimitry Andric       return PseudoOpc < TE.PseudoOpc;
1660b57cec5SDimitry Andric     }
1670b57cec5SDimitry Andric   };
1680b57cec5SDimitry Andric }
1690b57cec5SDimitry Andric 
1700b57cec5SDimitry Andric static const NEONLdStTableEntry NEONLdStTable[] = {
1710b57cec5SDimitry Andric { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
1720b57cec5SDimitry Andric { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
1730b57cec5SDimitry Andric { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
1740b57cec5SDimitry Andric { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
1750b57cec5SDimitry Andric { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
1760b57cec5SDimitry Andric { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
1770b57cec5SDimitry Andric 
1780b57cec5SDimitry Andric { ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
179fe6060f1SDimitry Andric { ARM::VLD1d16QPseudoWB_fixed,  ARM::VLD1d16Qwb_fixed,   true, true, false, SingleSpc,  4, 4 ,false},
180fe6060f1SDimitry Andric { ARM::VLD1d16QPseudoWB_register,  ARM::VLD1d16Qwb_register, true, true, true, SingleSpc,  4, 4 ,false},
1810b57cec5SDimitry Andric { ARM::VLD1d16TPseudo,      ARM::VLD1d16T,     true,  false, false, SingleSpc,  3, 4 ,false},
182fe6060f1SDimitry Andric { ARM::VLD1d16TPseudoWB_fixed,  ARM::VLD1d16Twb_fixed,   true, true, false, SingleSpc,  3, 4 ,false},
183fe6060f1SDimitry Andric { ARM::VLD1d16TPseudoWB_register,  ARM::VLD1d16Twb_register, true, true, true, SingleSpc,  3, 4 ,false},
184fe6060f1SDimitry Andric 
1850b57cec5SDimitry Andric { ARM::VLD1d32QPseudo,      ARM::VLD1d32Q,     true,  false, false, SingleSpc,  4, 2 ,false},
186fe6060f1SDimitry Andric { ARM::VLD1d32QPseudoWB_fixed,  ARM::VLD1d32Qwb_fixed,   true, true, false, SingleSpc,  4, 2 ,false},
187fe6060f1SDimitry Andric { ARM::VLD1d32QPseudoWB_register,  ARM::VLD1d32Qwb_register, true, true, true, SingleSpc,  4, 2 ,false},
1880b57cec5SDimitry Andric { ARM::VLD1d32TPseudo,      ARM::VLD1d32T,     true,  false, false, SingleSpc,  3, 2 ,false},
189fe6060f1SDimitry Andric { ARM::VLD1d32TPseudoWB_fixed,  ARM::VLD1d32Twb_fixed,   true, true, false, SingleSpc,  3, 2 ,false},
190fe6060f1SDimitry Andric { ARM::VLD1d32TPseudoWB_register,  ARM::VLD1d32Twb_register, true, true, true, SingleSpc,  3, 2 ,false},
191fe6060f1SDimitry Andric 
1920b57cec5SDimitry Andric { ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
1930b57cec5SDimitry Andric { ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
1940b57cec5SDimitry Andric { ARM::VLD1d64QPseudoWB_register,  ARM::VLD1d64Qwb_register,   true,  true, true, SingleSpc,  4, 1 ,false},
1950b57cec5SDimitry Andric { ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
1960b57cec5SDimitry Andric { ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
1970b57cec5SDimitry Andric { ARM::VLD1d64TPseudoWB_register,  ARM::VLD1d64Twb_register, true, true, true,  SingleSpc,  3, 1 ,false},
198fe6060f1SDimitry Andric 
1990b57cec5SDimitry Andric { ARM::VLD1d8QPseudo,       ARM::VLD1d8Q,      true,  false, false, SingleSpc,  4, 8 ,false},
200fe6060f1SDimitry Andric { ARM::VLD1d8QPseudoWB_fixed,   ARM::VLD1d8Qwb_fixed,    true,  true, false, SingleSpc,  4, 8 ,false},
201fe6060f1SDimitry Andric { ARM::VLD1d8QPseudoWB_register,   ARM::VLD1d8Qwb_register,  true, true, true, SingleSpc,  4, 8 ,false},
2020b57cec5SDimitry Andric { ARM::VLD1d8TPseudo,       ARM::VLD1d8T,      true,  false, false, SingleSpc,  3, 8 ,false},
203fe6060f1SDimitry Andric { ARM::VLD1d8TPseudoWB_fixed,   ARM::VLD1d8Twb_fixed,    true,  true, false, SingleSpc,  3, 8 ,false},
204fe6060f1SDimitry Andric { ARM::VLD1d8TPseudoWB_register,   ARM::VLD1d8Twb_register,  true,  true, true, SingleSpc,  3, 8 ,false},
205fe6060f1SDimitry Andric 
2060b57cec5SDimitry Andric { ARM::VLD1q16HighQPseudo,  ARM::VLD1d16Q,     true,  false, false, SingleHighQSpc,  4, 4 ,false},
207fe6060f1SDimitry Andric { ARM::VLD1q16HighQPseudo_UPD, ARM::VLD1d16Qwb_fixed,   true,  true, true, SingleHighQSpc,  4, 4 ,false},
2080b57cec5SDimitry Andric { ARM::VLD1q16HighTPseudo,  ARM::VLD1d16T,     true,  false, false, SingleHighTSpc,  3, 4 ,false},
209fe6060f1SDimitry Andric { ARM::VLD1q16HighTPseudo_UPD, ARM::VLD1d16Twb_fixed,   true,  true, true, SingleHighTSpc,  3, 4 ,false},
2100b57cec5SDimitry Andric { ARM::VLD1q16LowQPseudo_UPD,  ARM::VLD1d16Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 4 ,false},
2110b57cec5SDimitry Andric { ARM::VLD1q16LowTPseudo_UPD,  ARM::VLD1d16Twb_fixed,   true,  true, true, SingleLowSpc,  3, 4 ,false},
212fe6060f1SDimitry Andric 
2130b57cec5SDimitry Andric { ARM::VLD1q32HighQPseudo,  ARM::VLD1d32Q,     true,  false, false, SingleHighQSpc,  4, 2 ,false},
214fe6060f1SDimitry Andric { ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1d32Qwb_fixed,   true,  true, true, SingleHighQSpc,  4, 2 ,false},
2150b57cec5SDimitry Andric { ARM::VLD1q32HighTPseudo,  ARM::VLD1d32T,     true,  false, false, SingleHighTSpc,  3, 2 ,false},
216fe6060f1SDimitry Andric { ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1d32Twb_fixed,   true,  true, true, SingleHighTSpc,  3, 2 ,false},
2170b57cec5SDimitry Andric { ARM::VLD1q32LowQPseudo_UPD,  ARM::VLD1d32Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 2 ,false},
2180b57cec5SDimitry Andric { ARM::VLD1q32LowTPseudo_UPD,  ARM::VLD1d32Twb_fixed,   true,  true, true, SingleLowSpc,  3, 2 ,false},
219fe6060f1SDimitry Andric 
2200b57cec5SDimitry Andric { ARM::VLD1q64HighQPseudo,  ARM::VLD1d64Q,     true,  false, false, SingleHighQSpc,  4, 1 ,false},
221fe6060f1SDimitry Andric { ARM::VLD1q64HighQPseudo_UPD, ARM::VLD1d64Qwb_fixed,   true,  true, true, SingleHighQSpc,  4, 1 ,false},
2220b57cec5SDimitry Andric { ARM::VLD1q64HighTPseudo,  ARM::VLD1d64T,     true,  false, false, SingleHighTSpc,  3, 1 ,false},
223fe6060f1SDimitry Andric { ARM::VLD1q64HighTPseudo_UPD, ARM::VLD1d64Twb_fixed,   true,  true, true, SingleHighTSpc,  3, 1 ,false},
2240b57cec5SDimitry Andric { ARM::VLD1q64LowQPseudo_UPD,  ARM::VLD1d64Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 1 ,false},
2250b57cec5SDimitry Andric { ARM::VLD1q64LowTPseudo_UPD,  ARM::VLD1d64Twb_fixed,   true,  true, true, SingleLowSpc,  3, 1 ,false},
226fe6060f1SDimitry Andric 
2270b57cec5SDimitry Andric { ARM::VLD1q8HighQPseudo,   ARM::VLD1d8Q,     true,  false, false, SingleHighQSpc,  4, 8 ,false},
228fe6060f1SDimitry Andric { ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleHighQSpc,  4, 8 ,false},
2290b57cec5SDimitry Andric { ARM::VLD1q8HighTPseudo,   ARM::VLD1d8T,     true,  false, false, SingleHighTSpc,  3, 8 ,false},
230fe6060f1SDimitry Andric { ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1d8Twb_fixed,   true,  true, true, SingleHighTSpc,  3, 8 ,false},
2310b57cec5SDimitry Andric { ARM::VLD1q8LowQPseudo_UPD,  ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 8 ,false},
2320b57cec5SDimitry Andric { ARM::VLD1q8LowTPseudo_UPD,  ARM::VLD1d8Twb_fixed,   true,  true, true, SingleLowSpc,  3, 8 ,false},
2330b57cec5SDimitry Andric 
2340b57cec5SDimitry Andric { ARM::VLD2DUPq16EvenPseudo,  ARM::VLD2DUPd16x2,  true, false, false, EvenDblSpc, 2, 4 ,false},
2350b57cec5SDimitry Andric { ARM::VLD2DUPq16OddPseudo,   ARM::VLD2DUPd16x2,  true, false, false, OddDblSpc,  2, 4 ,false},
236fe6060f1SDimitry Andric { ARM::VLD2DUPq16OddPseudoWB_fixed,   ARM::VLD2DUPd16x2wb_fixed, true, true, false, OddDblSpc,  2, 4 ,false},
237fe6060f1SDimitry Andric { ARM::VLD2DUPq16OddPseudoWB_register,   ARM::VLD2DUPd16x2wb_register, true, true, true, OddDblSpc,  2, 4 ,false},
2380b57cec5SDimitry Andric { ARM::VLD2DUPq32EvenPseudo,  ARM::VLD2DUPd32x2,  true, false, false, EvenDblSpc, 2, 2 ,false},
2390b57cec5SDimitry Andric { ARM::VLD2DUPq32OddPseudo,   ARM::VLD2DUPd32x2,  true, false, false, OddDblSpc,  2, 2 ,false},
240fe6060f1SDimitry Andric { ARM::VLD2DUPq32OddPseudoWB_fixed,   ARM::VLD2DUPd32x2wb_fixed, true, true, false, OddDblSpc,  2, 2 ,false},
241fe6060f1SDimitry Andric { ARM::VLD2DUPq32OddPseudoWB_register,   ARM::VLD2DUPd32x2wb_register, true, true, true, OddDblSpc,  2, 2 ,false},
2420b57cec5SDimitry Andric { ARM::VLD2DUPq8EvenPseudo,   ARM::VLD2DUPd8x2,   true, false, false, EvenDblSpc, 2, 8 ,false},
2430b57cec5SDimitry Andric { ARM::VLD2DUPq8OddPseudo,    ARM::VLD2DUPd8x2,   true, false, false, OddDblSpc,  2, 8 ,false},
244fe6060f1SDimitry Andric { ARM::VLD2DUPq8OddPseudoWB_fixed,    ARM::VLD2DUPd8x2wb_fixed, true, true, false, OddDblSpc,  2, 8 ,false},
245fe6060f1SDimitry Andric { ARM::VLD2DUPq8OddPseudoWB_register,    ARM::VLD2DUPd8x2wb_register, true, true, true, OddDblSpc,  2, 8 ,false},
2460b57cec5SDimitry Andric 
2470b57cec5SDimitry Andric { ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
2480b57cec5SDimitry Andric { ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
2490b57cec5SDimitry Andric { ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
2500b57cec5SDimitry Andric { ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
2510b57cec5SDimitry Andric { ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
2520b57cec5SDimitry Andric { ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
2530b57cec5SDimitry Andric { ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},
2540b57cec5SDimitry Andric { ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true,  EvenDblSpc, 2, 4 ,true},
2550b57cec5SDimitry Andric { ARM::VLD2LNq32Pseudo,     ARM::VLD2LNq32,     true, false, false, EvenDblSpc, 2, 2 ,true},
2560b57cec5SDimitry Andric { ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true,  EvenDblSpc, 2, 2 ,true},
2570b57cec5SDimitry Andric 
2580b57cec5SDimitry Andric { ARM::VLD2q16Pseudo,       ARM::VLD2q16,      true,  false, false, SingleSpc,  4, 4 ,false},
2590b57cec5SDimitry Andric { ARM::VLD2q16PseudoWB_fixed,   ARM::VLD2q16wb_fixed, true, true, false,  SingleSpc,  4, 4 ,false},
2600b57cec5SDimitry Andric { ARM::VLD2q16PseudoWB_register,   ARM::VLD2q16wb_register, true, true, true,  SingleSpc,  4, 4 ,false},
2610b57cec5SDimitry Andric { ARM::VLD2q32Pseudo,       ARM::VLD2q32,      true,  false, false, SingleSpc,  4, 2 ,false},
2620b57cec5SDimitry Andric { ARM::VLD2q32PseudoWB_fixed,   ARM::VLD2q32wb_fixed, true, true, false,  SingleSpc,  4, 2 ,false},
2630b57cec5SDimitry Andric { ARM::VLD2q32PseudoWB_register,   ARM::VLD2q32wb_register, true, true, true,  SingleSpc,  4, 2 ,false},
2640b57cec5SDimitry Andric { ARM::VLD2q8Pseudo,        ARM::VLD2q8,       true,  false, false, SingleSpc,  4, 8 ,false},
2650b57cec5SDimitry Andric { ARM::VLD2q8PseudoWB_fixed,    ARM::VLD2q8wb_fixed, true, true, false,  SingleSpc,  4, 8 ,false},
2660b57cec5SDimitry Andric { ARM::VLD2q8PseudoWB_register,    ARM::VLD2q8wb_register, true, true, true,  SingleSpc,  4, 8 ,false},
2670b57cec5SDimitry Andric 
2680b57cec5SDimitry Andric { ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
2690b57cec5SDimitry Andric { ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
2700b57cec5SDimitry Andric { ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
2710b57cec5SDimitry Andric { ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
2720b57cec5SDimitry Andric { ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
2730b57cec5SDimitry Andric { ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
2740b57cec5SDimitry Andric { ARM::VLD3DUPq16EvenPseudo, ARM::VLD3DUPq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
2750b57cec5SDimitry Andric { ARM::VLD3DUPq16OddPseudo,  ARM::VLD3DUPq16,     true, false, false, OddDblSpc,  3, 4 ,true},
276fe6060f1SDimitry Andric { ARM::VLD3DUPq16OddPseudo_UPD,  ARM::VLD3DUPq16_UPD, true, true, true, OddDblSpc,  3, 4 ,true},
2770b57cec5SDimitry Andric { ARM::VLD3DUPq32EvenPseudo, ARM::VLD3DUPq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
2780b57cec5SDimitry Andric { ARM::VLD3DUPq32OddPseudo,  ARM::VLD3DUPq32,     true, false, false, OddDblSpc,  3, 2 ,true},
279fe6060f1SDimitry Andric { ARM::VLD3DUPq32OddPseudo_UPD,  ARM::VLD3DUPq32_UPD, true, true, true, OddDblSpc,  3, 2 ,true},
2800b57cec5SDimitry Andric { ARM::VLD3DUPq8EvenPseudo,  ARM::VLD3DUPq8,      true, false, false, EvenDblSpc, 3, 8 ,true},
2810b57cec5SDimitry Andric { ARM::VLD3DUPq8OddPseudo,   ARM::VLD3DUPq8,      true, false, false, OddDblSpc,  3, 8 ,true},
282fe6060f1SDimitry Andric { ARM::VLD3DUPq8OddPseudo_UPD,   ARM::VLD3DUPq8_UPD, true, true, true, OddDblSpc,  3, 8 ,true},
2830b57cec5SDimitry Andric 
2840b57cec5SDimitry Andric { ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
2850b57cec5SDimitry Andric { ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
2860b57cec5SDimitry Andric { ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
2870b57cec5SDimitry Andric { ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
2880b57cec5SDimitry Andric { ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
2890b57cec5SDimitry Andric { ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
2900b57cec5SDimitry Andric { ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
2910b57cec5SDimitry Andric { ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
2920b57cec5SDimitry Andric { ARM::VLD3LNq32Pseudo,     ARM::VLD3LNq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
2930b57cec5SDimitry Andric { ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
2940b57cec5SDimitry Andric 
2950b57cec5SDimitry Andric { ARM::VLD3d16Pseudo,       ARM::VLD3d16,      true,  false, false, SingleSpc,  3, 4 ,true},
2960b57cec5SDimitry Andric { ARM::VLD3d16Pseudo_UPD,   ARM::VLD3d16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
2970b57cec5SDimitry Andric { ARM::VLD3d32Pseudo,       ARM::VLD3d32,      true,  false, false, SingleSpc,  3, 2 ,true},
2980b57cec5SDimitry Andric { ARM::VLD3d32Pseudo_UPD,   ARM::VLD3d32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
2990b57cec5SDimitry Andric { ARM::VLD3d8Pseudo,        ARM::VLD3d8,       true,  false, false, SingleSpc,  3, 8 ,true},
3000b57cec5SDimitry Andric { ARM::VLD3d8Pseudo_UPD,    ARM::VLD3d8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
3010b57cec5SDimitry Andric 
3020b57cec5SDimitry Andric { ARM::VLD3q16Pseudo_UPD,    ARM::VLD3q16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
3030b57cec5SDimitry Andric { ARM::VLD3q16oddPseudo,     ARM::VLD3q16,     true,  false, false, OddDblSpc,  3, 4 ,true},
3040b57cec5SDimitry Andric { ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true,  OddDblSpc,  3, 4 ,true},
3050b57cec5SDimitry Andric { ARM::VLD3q32Pseudo_UPD,    ARM::VLD3q32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
3060b57cec5SDimitry Andric { ARM::VLD3q32oddPseudo,     ARM::VLD3q32,     true,  false, false, OddDblSpc,  3, 2 ,true},
3070b57cec5SDimitry Andric { ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true,  OddDblSpc,  3, 2 ,true},
3080b57cec5SDimitry Andric { ARM::VLD3q8Pseudo_UPD,     ARM::VLD3q8_UPD, true, true, true,  EvenDblSpc, 3, 8 ,true},
3090b57cec5SDimitry Andric { ARM::VLD3q8oddPseudo,      ARM::VLD3q8,      true,  false, false, OddDblSpc,  3, 8 ,true},
3100b57cec5SDimitry Andric { ARM::VLD3q8oddPseudo_UPD,  ARM::VLD3q8_UPD, true, true, true,  OddDblSpc,  3, 8 ,true},
3110b57cec5SDimitry Andric 
3120b57cec5SDimitry Andric { ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
3130b57cec5SDimitry Andric { ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
3140b57cec5SDimitry Andric { ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
3150b57cec5SDimitry Andric { ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
3160b57cec5SDimitry Andric { ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
3170b57cec5SDimitry Andric { ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
3180b57cec5SDimitry Andric { ARM::VLD4DUPq16EvenPseudo, ARM::VLD4DUPq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
3190b57cec5SDimitry Andric { ARM::VLD4DUPq16OddPseudo,  ARM::VLD4DUPq16,     true, false, false, OddDblSpc,  4, 4 ,true},
320fe6060f1SDimitry Andric { ARM::VLD4DUPq16OddPseudo_UPD,  ARM::VLD4DUPq16_UPD, true, true, true, OddDblSpc,  4, 4 ,true},
3210b57cec5SDimitry Andric { ARM::VLD4DUPq32EvenPseudo, ARM::VLD4DUPq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
3220b57cec5SDimitry Andric { ARM::VLD4DUPq32OddPseudo,  ARM::VLD4DUPq32,     true, false, false, OddDblSpc,  4, 2 ,true},
323fe6060f1SDimitry Andric { ARM::VLD4DUPq32OddPseudo_UPD,  ARM::VLD4DUPq32_UPD, true, true, true, OddDblSpc,  4, 2 ,true},
3240b57cec5SDimitry Andric { ARM::VLD4DUPq8EvenPseudo,  ARM::VLD4DUPq8,      true, false, false, EvenDblSpc, 4, 8 ,true},
3250b57cec5SDimitry Andric { ARM::VLD4DUPq8OddPseudo,   ARM::VLD4DUPq8,      true, false, false, OddDblSpc,  4, 8 ,true},
326fe6060f1SDimitry Andric { ARM::VLD4DUPq8OddPseudo_UPD,   ARM::VLD4DUPq8_UPD, true, true, true, OddDblSpc,  4, 8 ,true},
3270b57cec5SDimitry Andric 
3280b57cec5SDimitry Andric { ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
3290b57cec5SDimitry Andric { ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
3300b57cec5SDimitry Andric { ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
3310b57cec5SDimitry Andric { ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
3320b57cec5SDimitry Andric { ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
3330b57cec5SDimitry Andric { ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
3340b57cec5SDimitry Andric { ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
3350b57cec5SDimitry Andric { ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
3360b57cec5SDimitry Andric { ARM::VLD4LNq32Pseudo,     ARM::VLD4LNq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
3370b57cec5SDimitry Andric { ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
3380b57cec5SDimitry Andric 
3390b57cec5SDimitry Andric { ARM::VLD4d16Pseudo,       ARM::VLD4d16,      true,  false, false, SingleSpc,  4, 4 ,true},
3400b57cec5SDimitry Andric { ARM::VLD4d16Pseudo_UPD,   ARM::VLD4d16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
3410b57cec5SDimitry Andric { ARM::VLD4d32Pseudo,       ARM::VLD4d32,      true,  false, false, SingleSpc,  4, 2 ,true},
3420b57cec5SDimitry Andric { ARM::VLD4d32Pseudo_UPD,   ARM::VLD4d32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
3430b57cec5SDimitry Andric { ARM::VLD4d8Pseudo,        ARM::VLD4d8,       true,  false, false, SingleSpc,  4, 8 ,true},
3440b57cec5SDimitry Andric { ARM::VLD4d8Pseudo_UPD,    ARM::VLD4d8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
3450b57cec5SDimitry Andric 
3460b57cec5SDimitry Andric { ARM::VLD4q16Pseudo_UPD,    ARM::VLD4q16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
3470b57cec5SDimitry Andric { ARM::VLD4q16oddPseudo,     ARM::VLD4q16,     true,  false, false, OddDblSpc,  4, 4 ,true},
3480b57cec5SDimitry Andric { ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true,  OddDblSpc,  4, 4 ,true},
3490b57cec5SDimitry Andric { ARM::VLD4q32Pseudo_UPD,    ARM::VLD4q32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
3500b57cec5SDimitry Andric { ARM::VLD4q32oddPseudo,     ARM::VLD4q32,     true,  false, false, OddDblSpc,  4, 2 ,true},
3510b57cec5SDimitry Andric { ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true,  OddDblSpc,  4, 2 ,true},
3520b57cec5SDimitry Andric { ARM::VLD4q8Pseudo_UPD,     ARM::VLD4q8_UPD, true, true, true,  EvenDblSpc, 4, 8 ,true},
3530b57cec5SDimitry Andric { ARM::VLD4q8oddPseudo,      ARM::VLD4q8,      true,  false, false, OddDblSpc,  4, 8 ,true},
3540b57cec5SDimitry Andric { ARM::VLD4q8oddPseudo_UPD,  ARM::VLD4q8_UPD, true, true, true,  OddDblSpc,  4, 8 ,true},
3550b57cec5SDimitry Andric 
3560b57cec5SDimitry Andric { ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
3570b57cec5SDimitry Andric { ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
3580b57cec5SDimitry Andric { ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
3590b57cec5SDimitry Andric { ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
3600b57cec5SDimitry Andric { ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
3610b57cec5SDimitry Andric { ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
3620b57cec5SDimitry Andric 
3630b57cec5SDimitry Andric { ARM::VST1d16QPseudo,      ARM::VST1d16Q,     false, false, false, SingleSpc,  4, 4 ,false},
364fe6060f1SDimitry Andric { ARM::VST1d16QPseudoWB_fixed,  ARM::VST1d16Qwb_fixed, false, true, false, SingleSpc,  4, 4 ,false},
365fe6060f1SDimitry Andric { ARM::VST1d16QPseudoWB_register, ARM::VST1d16Qwb_register, false, true, true, SingleSpc,  4, 4 ,false},
3660b57cec5SDimitry Andric { ARM::VST1d16TPseudo,      ARM::VST1d16T,     false, false, false, SingleSpc,  3, 4 ,false},
367fe6060f1SDimitry Andric { ARM::VST1d16TPseudoWB_fixed,  ARM::VST1d16Twb_fixed, false, true, false, SingleSpc,  3, 4 ,false},
368fe6060f1SDimitry Andric { ARM::VST1d16TPseudoWB_register, ARM::VST1d16Twb_register, false, true, true, SingleSpc,  3, 4 ,false},
369fe6060f1SDimitry Andric 
3700b57cec5SDimitry Andric { ARM::VST1d32QPseudo,      ARM::VST1d32Q,     false, false, false, SingleSpc,  4, 2 ,false},
371fe6060f1SDimitry Andric { ARM::VST1d32QPseudoWB_fixed,  ARM::VST1d32Qwb_fixed, false, true, false, SingleSpc,  4, 2 ,false},
372fe6060f1SDimitry Andric { ARM::VST1d32QPseudoWB_register, ARM::VST1d32Qwb_register, false, true, true, SingleSpc,  4, 2 ,false},
3730b57cec5SDimitry Andric { ARM::VST1d32TPseudo,      ARM::VST1d32T,     false, false, false, SingleSpc,  3, 2 ,false},
374fe6060f1SDimitry Andric { ARM::VST1d32TPseudoWB_fixed,  ARM::VST1d32Twb_fixed, false, true, false, SingleSpc,  3, 2 ,false},
375fe6060f1SDimitry Andric { ARM::VST1d32TPseudoWB_register, ARM::VST1d32Twb_register, false, true, true, SingleSpc,  3, 2 ,false},
376fe6060f1SDimitry Andric 
3770b57cec5SDimitry Andric { ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
3780b57cec5SDimitry Andric { ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
3790b57cec5SDimitry Andric { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
3800b57cec5SDimitry Andric { ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
3810b57cec5SDimitry Andric { ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
3820b57cec5SDimitry Andric { ARM::VST1d64TPseudoWB_register, ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
383fe6060f1SDimitry Andric 
3840b57cec5SDimitry Andric { ARM::VST1d8QPseudo,       ARM::VST1d8Q,      false, false, false, SingleSpc,  4, 8 ,false},
385fe6060f1SDimitry Andric { ARM::VST1d8QPseudoWB_fixed,   ARM::VST1d8Qwb_fixed, false, true, false, SingleSpc,  4, 8 ,false},
386fe6060f1SDimitry Andric { ARM::VST1d8QPseudoWB_register,  ARM::VST1d8Qwb_register, false, true, true, SingleSpc,  4, 8 ,false},
3870b57cec5SDimitry Andric { ARM::VST1d8TPseudo,       ARM::VST1d8T,      false, false, false, SingleSpc,  3, 8 ,false},
388fe6060f1SDimitry Andric { ARM::VST1d8TPseudoWB_fixed,   ARM::VST1d8Twb_fixed, false, true, false, SingleSpc,  3, 8 ,false},
389fe6060f1SDimitry Andric { ARM::VST1d8TPseudoWB_register,  ARM::VST1d8Twb_register, false, true, true, SingleSpc,  3, 8 ,false},
390fe6060f1SDimitry Andric 
3910b57cec5SDimitry Andric { ARM::VST1q16HighQPseudo,  ARM::VST1d16Q,     false, false, false, SingleHighQSpc,   4, 4 ,false},
392fe6060f1SDimitry Andric { ARM::VST1q16HighQPseudo_UPD,  ARM::VST1d16Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
3930b57cec5SDimitry Andric { ARM::VST1q16HighTPseudo,  ARM::VST1d16T,     false, false, false, SingleHighTSpc,   3, 4 ,false},
394fe6060f1SDimitry Andric { ARM::VST1q16HighTPseudo_UPD,  ARM::VST1d16Twb_fixed,  false, true, true, SingleHighTSpc,   3, 4 ,false},
3950b57cec5SDimitry Andric { ARM::VST1q16LowQPseudo_UPD,   ARM::VST1d16Qwb_fixed,  false, true, true, SingleLowSpc,   4, 4 ,false},
3960b57cec5SDimitry Andric { ARM::VST1q16LowTPseudo_UPD,   ARM::VST1d16Twb_fixed,  false, true, true, SingleLowSpc,   3, 4 ,false},
397fe6060f1SDimitry Andric 
3980b57cec5SDimitry Andric { ARM::VST1q32HighQPseudo,  ARM::VST1d32Q,     false, false, false, SingleHighQSpc,   4, 2 ,false},
399fe6060f1SDimitry Andric { ARM::VST1q32HighQPseudo_UPD,  ARM::VST1d32Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
4000b57cec5SDimitry Andric { ARM::VST1q32HighTPseudo,  ARM::VST1d32T,     false, false, false, SingleHighTSpc,   3, 2 ,false},
401fe6060f1SDimitry Andric { ARM::VST1q32HighTPseudo_UPD,  ARM::VST1d32Twb_fixed,  false, true, true, SingleHighTSpc,   3, 2 ,false},
4020b57cec5SDimitry Andric { ARM::VST1q32LowQPseudo_UPD,   ARM::VST1d32Qwb_fixed,  false, true, true, SingleLowSpc,   4, 2 ,false},
4030b57cec5SDimitry Andric { ARM::VST1q32LowTPseudo_UPD,   ARM::VST1d32Twb_fixed,  false, true, true, SingleLowSpc,   3, 2 ,false},
404fe6060f1SDimitry Andric 
4050b57cec5SDimitry Andric { ARM::VST1q64HighQPseudo,  ARM::VST1d64Q,     false, false, false, SingleHighQSpc,   4, 1 ,false},
406fe6060f1SDimitry Andric { ARM::VST1q64HighQPseudo_UPD,  ARM::VST1d64Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
4070b57cec5SDimitry Andric { ARM::VST1q64HighTPseudo,  ARM::VST1d64T,     false, false, false, SingleHighTSpc,   3, 1 ,false},
408fe6060f1SDimitry Andric { ARM::VST1q64HighTPseudo_UPD,  ARM::VST1d64Twb_fixed,  false, true, true, SingleHighTSpc,   3, 1 ,false},
4090b57cec5SDimitry Andric { ARM::VST1q64LowQPseudo_UPD,   ARM::VST1d64Qwb_fixed,  false, true, true, SingleLowSpc,   4, 1 ,false},
4100b57cec5SDimitry Andric { ARM::VST1q64LowTPseudo_UPD,   ARM::VST1d64Twb_fixed,  false, true, true, SingleLowSpc,   3, 1 ,false},
411fe6060f1SDimitry Andric 
4120b57cec5SDimitry Andric { ARM::VST1q8HighQPseudo,   ARM::VST1d8Q,      false, false, false, SingleHighQSpc,   4, 8 ,false},
413fe6060f1SDimitry Andric { ARM::VST1q8HighQPseudo_UPD,  ARM::VST1d8Qwb_fixed,  false, true, true, SingleHighQSpc,   4, 8 ,false},
4140b57cec5SDimitry Andric { ARM::VST1q8HighTPseudo,   ARM::VST1d8T,      false, false, false, SingleHighTSpc,   3, 8 ,false},
415fe6060f1SDimitry Andric { ARM::VST1q8HighTPseudo_UPD,  ARM::VST1d8Twb_fixed,  false, true, true, SingleHighTSpc,   3, 8 ,false},
4160b57cec5SDimitry Andric { ARM::VST1q8LowQPseudo_UPD,   ARM::VST1d8Qwb_fixed,  false, true, true, SingleLowSpc,   4, 8 ,false},
4170b57cec5SDimitry Andric { ARM::VST1q8LowTPseudo_UPD,   ARM::VST1d8Twb_fixed,  false, true, true, SingleLowSpc,   3, 8 ,false},
4180b57cec5SDimitry Andric 
4190b57cec5SDimitry Andric { ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
4200b57cec5SDimitry Andric { ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
4210b57cec5SDimitry Andric { ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
4220b57cec5SDimitry Andric { ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
4230b57cec5SDimitry Andric { ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
4240b57cec5SDimitry Andric { ARM::VST2LNd8Pseudo_UPD,  ARM::VST2LNd8_UPD, false, true, true,  SingleSpc, 2, 8 ,true},
4250b57cec5SDimitry Andric { ARM::VST2LNq16Pseudo,     ARM::VST2LNq16,     false, false, false, EvenDblSpc, 2, 4,true},
4260b57cec5SDimitry Andric { ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true,  EvenDblSpc, 2, 4,true},
4270b57cec5SDimitry Andric { ARM::VST2LNq32Pseudo,     ARM::VST2LNq32,     false, false, false, EvenDblSpc, 2, 2,true},
4280b57cec5SDimitry Andric { ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true,  EvenDblSpc, 2, 2,true},
4290b57cec5SDimitry Andric 
4300b57cec5SDimitry Andric { ARM::VST2q16Pseudo,       ARM::VST2q16,      false, false, false, SingleSpc,  4, 4 ,false},
4310b57cec5SDimitry Andric { ARM::VST2q16PseudoWB_fixed,   ARM::VST2q16wb_fixed, false, true, false,  SingleSpc,  4, 4 ,false},
4320b57cec5SDimitry Andric { ARM::VST2q16PseudoWB_register,   ARM::VST2q16wb_register, false, true, true,  SingleSpc,  4, 4 ,false},
4330b57cec5SDimitry Andric { ARM::VST2q32Pseudo,       ARM::VST2q32,      false, false, false, SingleSpc,  4, 2 ,false},
4340b57cec5SDimitry Andric { ARM::VST2q32PseudoWB_fixed,   ARM::VST2q32wb_fixed, false, true, false,  SingleSpc,  4, 2 ,false},
4350b57cec5SDimitry Andric { ARM::VST2q32PseudoWB_register,   ARM::VST2q32wb_register, false, true, true,  SingleSpc,  4, 2 ,false},
4360b57cec5SDimitry Andric { ARM::VST2q8Pseudo,        ARM::VST2q8,       false, false, false, SingleSpc,  4, 8 ,false},
4370b57cec5SDimitry Andric { ARM::VST2q8PseudoWB_fixed,    ARM::VST2q8wb_fixed, false, true, false,  SingleSpc,  4, 8 ,false},
4380b57cec5SDimitry Andric { ARM::VST2q8PseudoWB_register,    ARM::VST2q8wb_register, false, true, true,  SingleSpc,  4, 8 ,false},
4390b57cec5SDimitry Andric 
4400b57cec5SDimitry Andric { ARM::VST3LNd16Pseudo,     ARM::VST3LNd16,     false, false, false, SingleSpc, 3, 4 ,true},
4410b57cec5SDimitry Andric { ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true,  SingleSpc, 3, 4 ,true},
4420b57cec5SDimitry Andric { ARM::VST3LNd32Pseudo,     ARM::VST3LNd32,     false, false, false, SingleSpc, 3, 2 ,true},
4430b57cec5SDimitry Andric { ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true,  SingleSpc, 3, 2 ,true},
4440b57cec5SDimitry Andric { ARM::VST3LNd8Pseudo,      ARM::VST3LNd8,      false, false, false, SingleSpc, 3, 8 ,true},
4450b57cec5SDimitry Andric { ARM::VST3LNd8Pseudo_UPD,  ARM::VST3LNd8_UPD, false, true, true,  SingleSpc, 3, 8 ,true},
4460b57cec5SDimitry Andric { ARM::VST3LNq16Pseudo,     ARM::VST3LNq16,     false, false, false, EvenDblSpc, 3, 4,true},
4470b57cec5SDimitry Andric { ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true,  EvenDblSpc, 3, 4,true},
4480b57cec5SDimitry Andric { ARM::VST3LNq32Pseudo,     ARM::VST3LNq32,     false, false, false, EvenDblSpc, 3, 2,true},
4490b57cec5SDimitry Andric { ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true,  EvenDblSpc, 3, 2,true},
4500b57cec5SDimitry Andric 
4510b57cec5SDimitry Andric { ARM::VST3d16Pseudo,       ARM::VST3d16,      false, false, false, SingleSpc,  3, 4 ,true},
4520b57cec5SDimitry Andric { ARM::VST3d16Pseudo_UPD,   ARM::VST3d16_UPD, false, true, true,  SingleSpc,  3, 4 ,true},
4530b57cec5SDimitry Andric { ARM::VST3d32Pseudo,       ARM::VST3d32,      false, false, false, SingleSpc,  3, 2 ,true},
4540b57cec5SDimitry Andric { ARM::VST3d32Pseudo_UPD,   ARM::VST3d32_UPD, false, true, true,  SingleSpc,  3, 2 ,true},
4550b57cec5SDimitry Andric { ARM::VST3d8Pseudo,        ARM::VST3d8,       false, false, false, SingleSpc,  3, 8 ,true},
4560b57cec5SDimitry Andric { ARM::VST3d8Pseudo_UPD,    ARM::VST3d8_UPD, false, true, true,  SingleSpc,  3, 8 ,true},
4570b57cec5SDimitry Andric 
4580b57cec5SDimitry Andric { ARM::VST3q16Pseudo_UPD,    ARM::VST3q16_UPD, false, true, true,  EvenDblSpc, 3, 4 ,true},
4590b57cec5SDimitry Andric { ARM::VST3q16oddPseudo,     ARM::VST3q16,     false, false, false, OddDblSpc,  3, 4 ,true},
4600b57cec5SDimitry Andric { ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true,  OddDblSpc,  3, 4 ,true},
4610b57cec5SDimitry Andric { ARM::VST3q32Pseudo_UPD,    ARM::VST3q32_UPD, false, true, true,  EvenDblSpc, 3, 2 ,true},
4620b57cec5SDimitry Andric { ARM::VST3q32oddPseudo,     ARM::VST3q32,     false, false, false, OddDblSpc,  3, 2 ,true},
4630b57cec5SDimitry Andric { ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true,  OddDblSpc,  3, 2 ,true},
4640b57cec5SDimitry Andric { ARM::VST3q8Pseudo_UPD,     ARM::VST3q8_UPD, false, true, true,  EvenDblSpc, 3, 8 ,true},
4650b57cec5SDimitry Andric { ARM::VST3q8oddPseudo,      ARM::VST3q8,      false, false, false, OddDblSpc,  3, 8 ,true},
4660b57cec5SDimitry Andric { ARM::VST3q8oddPseudo_UPD,  ARM::VST3q8_UPD, false, true, true,  OddDblSpc,  3, 8 ,true},
4670b57cec5SDimitry Andric 
4680b57cec5SDimitry Andric { ARM::VST4LNd16Pseudo,     ARM::VST4LNd16,     false, false, false, SingleSpc, 4, 4 ,true},
4690b57cec5SDimitry Andric { ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true,  SingleSpc, 4, 4 ,true},
4700b57cec5SDimitry Andric { ARM::VST4LNd32Pseudo,     ARM::VST4LNd32,     false, false, false, SingleSpc, 4, 2 ,true},
4710b57cec5SDimitry Andric { ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true,  SingleSpc, 4, 2 ,true},
4720b57cec5SDimitry Andric { ARM::VST4LNd8Pseudo,      ARM::VST4LNd8,      false, false, false, SingleSpc, 4, 8 ,true},
4730b57cec5SDimitry Andric { ARM::VST4LNd8Pseudo_UPD,  ARM::VST4LNd8_UPD, false, true, true,  SingleSpc, 4, 8 ,true},
4740b57cec5SDimitry Andric { ARM::VST4LNq16Pseudo,     ARM::VST4LNq16,     false, false, false, EvenDblSpc, 4, 4,true},
4750b57cec5SDimitry Andric { ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true,  EvenDblSpc, 4, 4,true},
4760b57cec5SDimitry Andric { ARM::VST4LNq32Pseudo,     ARM::VST4LNq32,     false, false, false, EvenDblSpc, 4, 2,true},
4770b57cec5SDimitry Andric { ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true,  EvenDblSpc, 4, 2,true},
4780b57cec5SDimitry Andric 
4790b57cec5SDimitry Andric { ARM::VST4d16Pseudo,       ARM::VST4d16,      false, false, false, SingleSpc,  4, 4 ,true},
4800b57cec5SDimitry Andric { ARM::VST4d16Pseudo_UPD,   ARM::VST4d16_UPD, false, true, true,  SingleSpc,  4, 4 ,true},
4810b57cec5SDimitry Andric { ARM::VST4d32Pseudo,       ARM::VST4d32,      false, false, false, SingleSpc,  4, 2 ,true},
4820b57cec5SDimitry Andric { ARM::VST4d32Pseudo_UPD,   ARM::VST4d32_UPD, false, true, true,  SingleSpc,  4, 2 ,true},
4830b57cec5SDimitry Andric { ARM::VST4d8Pseudo,        ARM::VST4d8,       false, false, false, SingleSpc,  4, 8 ,true},
4840b57cec5SDimitry Andric { ARM::VST4d8Pseudo_UPD,    ARM::VST4d8_UPD, false, true, true,  SingleSpc,  4, 8 ,true},
4850b57cec5SDimitry Andric 
4860b57cec5SDimitry Andric { ARM::VST4q16Pseudo_UPD,    ARM::VST4q16_UPD, false, true, true,  EvenDblSpc, 4, 4 ,true},
4870b57cec5SDimitry Andric { ARM::VST4q16oddPseudo,     ARM::VST4q16,     false, false, false, OddDblSpc,  4, 4 ,true},
4880b57cec5SDimitry Andric { ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true,  OddDblSpc,  4, 4 ,true},
4890b57cec5SDimitry Andric { ARM::VST4q32Pseudo_UPD,    ARM::VST4q32_UPD, false, true, true,  EvenDblSpc, 4, 2 ,true},
4900b57cec5SDimitry Andric { ARM::VST4q32oddPseudo,     ARM::VST4q32,     false, false, false, OddDblSpc,  4, 2 ,true},
4910b57cec5SDimitry Andric { ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true,  OddDblSpc,  4, 2 ,true},
4920b57cec5SDimitry Andric { ARM::VST4q8Pseudo_UPD,     ARM::VST4q8_UPD, false, true, true,  EvenDblSpc, 4, 8 ,true},
4930b57cec5SDimitry Andric { ARM::VST4q8oddPseudo,      ARM::VST4q8,      false, false, false, OddDblSpc,  4, 8 ,true},
4940b57cec5SDimitry Andric { ARM::VST4q8oddPseudo_UPD,  ARM::VST4q8_UPD, false, true, true,  OddDblSpc,  4, 8 ,true}
4950b57cec5SDimitry Andric };
4960b57cec5SDimitry Andric 
4970b57cec5SDimitry Andric /// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
4980b57cec5SDimitry Andric /// load or store pseudo instruction.
4990b57cec5SDimitry Andric static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
5000b57cec5SDimitry Andric #ifndef NDEBUG
5010b57cec5SDimitry Andric   // Make sure the table is sorted.
5020b57cec5SDimitry Andric   static std::atomic<bool> TableChecked(false);
5030b57cec5SDimitry Andric   if (!TableChecked.load(std::memory_order_relaxed)) {
5045ffd83dbSDimitry Andric     assert(llvm::is_sorted(NEONLdStTable) && "NEONLdStTable is not sorted!");
5050b57cec5SDimitry Andric     TableChecked.store(true, std::memory_order_relaxed);
5060b57cec5SDimitry Andric   }
5070b57cec5SDimitry Andric #endif
5080b57cec5SDimitry Andric 
5090b57cec5SDimitry Andric   auto I = llvm::lower_bound(NEONLdStTable, Opcode);
5100b57cec5SDimitry Andric   if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)
5110b57cec5SDimitry Andric     return I;
5120b57cec5SDimitry Andric   return nullptr;
5130b57cec5SDimitry Andric }
5140b57cec5SDimitry Andric 
5150b57cec5SDimitry Andric /// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
5160b57cec5SDimitry Andric /// corresponding to the specified register spacing.  Not all of the results
5170b57cec5SDimitry Andric /// are necessarily valid, e.g., a Q register only has 2 D subregisters.
5180b57cec5SDimitry Andric static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
5190b57cec5SDimitry Andric                         const TargetRegisterInfo *TRI, unsigned &D0,
5200b57cec5SDimitry Andric                         unsigned &D1, unsigned &D2, unsigned &D3) {
5210b57cec5SDimitry Andric   if (RegSpc == SingleSpc || RegSpc == SingleLowSpc) {
5220b57cec5SDimitry Andric     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
5230b57cec5SDimitry Andric     D1 = TRI->getSubReg(Reg, ARM::dsub_1);
5240b57cec5SDimitry Andric     D2 = TRI->getSubReg(Reg, ARM::dsub_2);
5250b57cec5SDimitry Andric     D3 = TRI->getSubReg(Reg, ARM::dsub_3);
5260b57cec5SDimitry Andric   } else if (RegSpc == SingleHighQSpc) {
5270b57cec5SDimitry Andric     D0 = TRI->getSubReg(Reg, ARM::dsub_4);
5280b57cec5SDimitry Andric     D1 = TRI->getSubReg(Reg, ARM::dsub_5);
5290b57cec5SDimitry Andric     D2 = TRI->getSubReg(Reg, ARM::dsub_6);
5300b57cec5SDimitry Andric     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
5310b57cec5SDimitry Andric   } else if (RegSpc == SingleHighTSpc) {
5320b57cec5SDimitry Andric     D0 = TRI->getSubReg(Reg, ARM::dsub_3);
5330b57cec5SDimitry Andric     D1 = TRI->getSubReg(Reg, ARM::dsub_4);
5340b57cec5SDimitry Andric     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
5350b57cec5SDimitry Andric     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
5360b57cec5SDimitry Andric   } else if (RegSpc == EvenDblSpc) {
5370b57cec5SDimitry Andric     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
5380b57cec5SDimitry Andric     D1 = TRI->getSubReg(Reg, ARM::dsub_2);
5390b57cec5SDimitry Andric     D2 = TRI->getSubReg(Reg, ARM::dsub_4);
5400b57cec5SDimitry Andric     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
5410b57cec5SDimitry Andric   } else {
5420b57cec5SDimitry Andric     assert(RegSpc == OddDblSpc && "unknown register spacing");
5430b57cec5SDimitry Andric     D0 = TRI->getSubReg(Reg, ARM::dsub_1);
5440b57cec5SDimitry Andric     D1 = TRI->getSubReg(Reg, ARM::dsub_3);
5450b57cec5SDimitry Andric     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
5460b57cec5SDimitry Andric     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
5470b57cec5SDimitry Andric   }
5480b57cec5SDimitry Andric }
5490b57cec5SDimitry Andric 
5500b57cec5SDimitry Andric /// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
5510b57cec5SDimitry Andric /// operands to real VLD instructions with D register operands.
5520b57cec5SDimitry Andric void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
5530b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
5540b57cec5SDimitry Andric   MachineBasicBlock &MBB = *MI.getParent();
5550b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
5560b57cec5SDimitry Andric 
5570b57cec5SDimitry Andric   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
5580b57cec5SDimitry Andric   assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
5590b57cec5SDimitry Andric   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
5600b57cec5SDimitry Andric   unsigned NumRegs = TableEntry->NumRegs;
5610b57cec5SDimitry Andric 
5620b57cec5SDimitry Andric   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
5630b57cec5SDimitry Andric                                     TII->get(TableEntry->RealOpc));
5640b57cec5SDimitry Andric   unsigned OpIdx = 0;
5650b57cec5SDimitry Andric 
5660b57cec5SDimitry Andric   bool DstIsDead = MI.getOperand(OpIdx).isDead();
5678bcb0991SDimitry Andric   Register DstReg = MI.getOperand(OpIdx++).getReg();
568fe6060f1SDimitry Andric 
569fe6060f1SDimitry Andric   bool IsVLD2DUP = TableEntry->RealOpc == ARM::VLD2DUPd8x2 ||
5700b57cec5SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd16x2 ||
571fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd32x2 ||
572fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||
573fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||
574fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed ||
575fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_register ||
576fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_register ||
577fe6060f1SDimitry Andric                    TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_register;
578fe6060f1SDimitry Andric 
579fe6060f1SDimitry Andric   if (IsVLD2DUP) {
5800b57cec5SDimitry Andric     unsigned SubRegIndex;
5810b57cec5SDimitry Andric     if (RegSpc == EvenDblSpc) {
5820b57cec5SDimitry Andric       SubRegIndex = ARM::dsub_0;
5830b57cec5SDimitry Andric     } else {
5840b57cec5SDimitry Andric       assert(RegSpc == OddDblSpc && "Unexpected spacing!");
5850b57cec5SDimitry Andric       SubRegIndex = ARM::dsub_1;
5860b57cec5SDimitry Andric     }
5878bcb0991SDimitry Andric     Register SubReg = TRI->getSubReg(DstReg, SubRegIndex);
5880b57cec5SDimitry Andric     unsigned DstRegPair = TRI->getMatchingSuperReg(SubReg, ARM::dsub_0,
5890b57cec5SDimitry Andric                                                    &ARM::DPairSpcRegClass);
5900b57cec5SDimitry Andric     MIB.addReg(DstRegPair, RegState::Define | getDeadRegState(DstIsDead));
5910b57cec5SDimitry Andric   } else {
5920b57cec5SDimitry Andric     unsigned D0, D1, D2, D3;
5930b57cec5SDimitry Andric     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
5940b57cec5SDimitry Andric     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
5950b57cec5SDimitry Andric     if (NumRegs > 1 && TableEntry->copyAllListRegs)
5960b57cec5SDimitry Andric       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
5970b57cec5SDimitry Andric     if (NumRegs > 2 && TableEntry->copyAllListRegs)
5980b57cec5SDimitry Andric       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
5990b57cec5SDimitry Andric     if (NumRegs > 3 && TableEntry->copyAllListRegs)
6000b57cec5SDimitry Andric       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
6010b57cec5SDimitry Andric   }
6020b57cec5SDimitry Andric 
6030b57cec5SDimitry Andric   if (TableEntry->isUpdating)
6040b57cec5SDimitry Andric     MIB.add(MI.getOperand(OpIdx++));
6050b57cec5SDimitry Andric 
6060b57cec5SDimitry Andric   // Copy the addrmode6 operands.
6070b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
6080b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
6090b57cec5SDimitry Andric 
6100b57cec5SDimitry Andric   // Copy the am6offset operand.
6110b57cec5SDimitry Andric   if (TableEntry->hasWritebackOperand) {
6120b57cec5SDimitry Andric     // TODO: The writing-back pseudo instructions we translate here are all
6130b57cec5SDimitry Andric     // defined to take am6offset nodes that are capable to represent both fixed
6140b57cec5SDimitry Andric     // and register forms. Some real instructions, however, do not rely on
6150b57cec5SDimitry Andric     // am6offset and have separate definitions for such forms. When this is the
6160b57cec5SDimitry Andric     // case, fixed forms do not take any offset nodes, so here we skip them for
6170b57cec5SDimitry Andric     // such instructions. Once all real and pseudo writing-back instructions are
6180b57cec5SDimitry Andric     // rewritten without use of am6offset nodes, this code will go away.
6190b57cec5SDimitry Andric     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
6200b57cec5SDimitry Andric     if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||
6210b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed ||
6220b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed ||
6230b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed ||
6240b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d8Twb_fixed ||
6250b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d16Twb_fixed ||
6260b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d32Twb_fixed ||
627fe6060f1SDimitry Andric         TableEntry->RealOpc == ARM::VLD1d64Twb_fixed ||
628fe6060f1SDimitry Andric         TableEntry->RealOpc == ARM::VLD2DUPd8x2wb_fixed ||
629fe6060f1SDimitry Andric         TableEntry->RealOpc == ARM::VLD2DUPd16x2wb_fixed ||
630fe6060f1SDimitry Andric         TableEntry->RealOpc == ARM::VLD2DUPd32x2wb_fixed) {
6310b57cec5SDimitry Andric       assert(AM6Offset.getReg() == 0 &&
6320b57cec5SDimitry Andric              "A fixed writing-back pseudo instruction provides an offset "
6330b57cec5SDimitry Andric              "register!");
6340b57cec5SDimitry Andric     } else {
6350b57cec5SDimitry Andric       MIB.add(AM6Offset);
6360b57cec5SDimitry Andric     }
6370b57cec5SDimitry Andric   }
6380b57cec5SDimitry Andric 
6390b57cec5SDimitry Andric   // For an instruction writing double-spaced subregs, the pseudo instruction
6400b57cec5SDimitry Andric   // has an extra operand that is a use of the super-register.  Record the
6410b57cec5SDimitry Andric   // operand index and skip over it.
6420b57cec5SDimitry Andric   unsigned SrcOpIdx = 0;
643*0fca6ea1SDimitry Andric   if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc || RegSpc == SingleLowSpc ||
644*0fca6ea1SDimitry Andric       RegSpc == SingleHighQSpc || RegSpc == SingleHighTSpc)
6450b57cec5SDimitry Andric     SrcOpIdx = OpIdx++;
6460b57cec5SDimitry Andric 
6470b57cec5SDimitry Andric   // Copy the predicate operands.
6480b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
6490b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
6500b57cec5SDimitry Andric 
6510b57cec5SDimitry Andric   // Copy the super-register source operand used for double-spaced subregs over
6520b57cec5SDimitry Andric   // to the new instruction as an implicit operand.
6530b57cec5SDimitry Andric   if (SrcOpIdx != 0) {
6540b57cec5SDimitry Andric     MachineOperand MO = MI.getOperand(SrcOpIdx);
6550b57cec5SDimitry Andric     MO.setImplicit(true);
6560b57cec5SDimitry Andric     MIB.add(MO);
6570b57cec5SDimitry Andric   }
6580b57cec5SDimitry Andric   // Add an implicit def for the super-register.
6590b57cec5SDimitry Andric   MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
66006c3fb27SDimitry Andric   MIB.copyImplicitOps(MI);
6610b57cec5SDimitry Andric 
6620b57cec5SDimitry Andric   // Transfer memoperands.
6630b57cec5SDimitry Andric   MIB.cloneMemRefs(MI);
6640b57cec5SDimitry Andric   MI.eraseFromParent();
6650b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "To:        "; MIB.getInstr()->dump(););
6660b57cec5SDimitry Andric }
6670b57cec5SDimitry Andric 
6680b57cec5SDimitry Andric /// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
6690b57cec5SDimitry Andric /// operands to real VST instructions with D register operands.
6700b57cec5SDimitry Andric void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
6710b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
6720b57cec5SDimitry Andric   MachineBasicBlock &MBB = *MI.getParent();
6730b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
6740b57cec5SDimitry Andric 
6750b57cec5SDimitry Andric   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
6760b57cec5SDimitry Andric   assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
6770b57cec5SDimitry Andric   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
6780b57cec5SDimitry Andric   unsigned NumRegs = TableEntry->NumRegs;
6790b57cec5SDimitry Andric 
6800b57cec5SDimitry Andric   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
6810b57cec5SDimitry Andric                                     TII->get(TableEntry->RealOpc));
6820b57cec5SDimitry Andric   unsigned OpIdx = 0;
6830b57cec5SDimitry Andric   if (TableEntry->isUpdating)
6840b57cec5SDimitry Andric     MIB.add(MI.getOperand(OpIdx++));
6850b57cec5SDimitry Andric 
6860b57cec5SDimitry Andric   // Copy the addrmode6 operands.
6870b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
6880b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
6890b57cec5SDimitry Andric 
6900b57cec5SDimitry Andric   if (TableEntry->hasWritebackOperand) {
6910b57cec5SDimitry Andric     // TODO: The writing-back pseudo instructions we translate here are all
6920b57cec5SDimitry Andric     // defined to take am6offset nodes that are capable to represent both fixed
6930b57cec5SDimitry Andric     // and register forms. Some real instructions, however, do not rely on
6940b57cec5SDimitry Andric     // am6offset and have separate definitions for such forms. When this is the
6950b57cec5SDimitry Andric     // case, fixed forms do not take any offset nodes, so here we skip them for
6960b57cec5SDimitry Andric     // such instructions. Once all real and pseudo writing-back instructions are
6970b57cec5SDimitry Andric     // rewritten without use of am6offset nodes, this code will go away.
6980b57cec5SDimitry Andric     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
6990b57cec5SDimitry Andric     if (TableEntry->RealOpc == ARM::VST1d8Qwb_fixed ||
7000b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d16Qwb_fixed ||
7010b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d32Qwb_fixed ||
7020b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d64Qwb_fixed ||
7030b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d8Twb_fixed ||
7040b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d16Twb_fixed ||
7050b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d32Twb_fixed ||
7060b57cec5SDimitry Andric         TableEntry->RealOpc == ARM::VST1d64Twb_fixed) {
7070b57cec5SDimitry Andric       assert(AM6Offset.getReg() == 0 &&
7080b57cec5SDimitry Andric              "A fixed writing-back pseudo instruction provides an offset "
7090b57cec5SDimitry Andric              "register!");
7100b57cec5SDimitry Andric     } else {
7110b57cec5SDimitry Andric       MIB.add(AM6Offset);
7120b57cec5SDimitry Andric     }
7130b57cec5SDimitry Andric   }
7140b57cec5SDimitry Andric 
7150b57cec5SDimitry Andric   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
7160b57cec5SDimitry Andric   bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();
7178bcb0991SDimitry Andric   Register SrcReg = MI.getOperand(OpIdx++).getReg();
7180b57cec5SDimitry Andric   unsigned D0, D1, D2, D3;
7190b57cec5SDimitry Andric   GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);
7200b57cec5SDimitry Andric   MIB.addReg(D0, getUndefRegState(SrcIsUndef));
7210b57cec5SDimitry Andric   if (NumRegs > 1 && TableEntry->copyAllListRegs)
7220b57cec5SDimitry Andric     MIB.addReg(D1, getUndefRegState(SrcIsUndef));
7230b57cec5SDimitry Andric   if (NumRegs > 2 && TableEntry->copyAllListRegs)
7240b57cec5SDimitry Andric     MIB.addReg(D2, getUndefRegState(SrcIsUndef));
7250b57cec5SDimitry Andric   if (NumRegs > 3 && TableEntry->copyAllListRegs)
7260b57cec5SDimitry Andric     MIB.addReg(D3, getUndefRegState(SrcIsUndef));
7270b57cec5SDimitry Andric 
7280b57cec5SDimitry Andric   // Copy the predicate operands.
7290b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
7300b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
7310b57cec5SDimitry Andric 
7320b57cec5SDimitry Andric   if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.
7330b57cec5SDimitry Andric     MIB->addRegisterKilled(SrcReg, TRI, true);
7340b57cec5SDimitry Andric   else if (!SrcIsUndef)
7350b57cec5SDimitry Andric     MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.
73606c3fb27SDimitry Andric   MIB.copyImplicitOps(MI);
7370b57cec5SDimitry Andric 
7380b57cec5SDimitry Andric   // Transfer memoperands.
7390b57cec5SDimitry Andric   MIB.cloneMemRefs(MI);
7400b57cec5SDimitry Andric   MI.eraseFromParent();
7410b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "To:        "; MIB.getInstr()->dump(););
7420b57cec5SDimitry Andric }
7430b57cec5SDimitry Andric 
7440b57cec5SDimitry Andric /// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
7450b57cec5SDimitry Andric /// register operands to real instructions with D register operands.
7460b57cec5SDimitry Andric void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
7470b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
7480b57cec5SDimitry Andric   MachineBasicBlock &MBB = *MI.getParent();
7490b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
7500b57cec5SDimitry Andric 
7510b57cec5SDimitry Andric   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
7520b57cec5SDimitry Andric   assert(TableEntry && "NEONLdStTable lookup failed");
7530b57cec5SDimitry Andric   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
7540b57cec5SDimitry Andric   unsigned NumRegs = TableEntry->NumRegs;
7550b57cec5SDimitry Andric   unsigned RegElts = TableEntry->RegElts;
7560b57cec5SDimitry Andric 
7570b57cec5SDimitry Andric   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
7580b57cec5SDimitry Andric                                     TII->get(TableEntry->RealOpc));
7590b57cec5SDimitry Andric   unsigned OpIdx = 0;
7600b57cec5SDimitry Andric   // The lane operand is always the 3rd from last operand, before the 2
7610b57cec5SDimitry Andric   // predicate operands.
7620b57cec5SDimitry Andric   unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();
7630b57cec5SDimitry Andric 
7640b57cec5SDimitry Andric   // Adjust the lane and spacing as needed for Q registers.
7650b57cec5SDimitry Andric   assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
7660b57cec5SDimitry Andric   if (RegSpc == EvenDblSpc && Lane >= RegElts) {
7670b57cec5SDimitry Andric     RegSpc = OddDblSpc;
7680b57cec5SDimitry Andric     Lane -= RegElts;
7690b57cec5SDimitry Andric   }
7700b57cec5SDimitry Andric   assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
7710b57cec5SDimitry Andric 
7720b57cec5SDimitry Andric   unsigned D0 = 0, D1 = 0, D2 = 0, D3 = 0;
7730b57cec5SDimitry Andric   unsigned DstReg = 0;
7740b57cec5SDimitry Andric   bool DstIsDead = false;
7750b57cec5SDimitry Andric   if (TableEntry->IsLoad) {
7760b57cec5SDimitry Andric     DstIsDead = MI.getOperand(OpIdx).isDead();
7770b57cec5SDimitry Andric     DstReg = MI.getOperand(OpIdx++).getReg();
7780b57cec5SDimitry Andric     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
7790b57cec5SDimitry Andric     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
7800b57cec5SDimitry Andric     if (NumRegs > 1)
7810b57cec5SDimitry Andric       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
7820b57cec5SDimitry Andric     if (NumRegs > 2)
7830b57cec5SDimitry Andric       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
7840b57cec5SDimitry Andric     if (NumRegs > 3)
7850b57cec5SDimitry Andric       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
7860b57cec5SDimitry Andric   }
7870b57cec5SDimitry Andric 
7880b57cec5SDimitry Andric   if (TableEntry->isUpdating)
7890b57cec5SDimitry Andric     MIB.add(MI.getOperand(OpIdx++));
7900b57cec5SDimitry Andric 
7910b57cec5SDimitry Andric   // Copy the addrmode6 operands.
7920b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
7930b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
7940b57cec5SDimitry Andric   // Copy the am6offset operand.
7950b57cec5SDimitry Andric   if (TableEntry->hasWritebackOperand)
7960b57cec5SDimitry Andric     MIB.add(MI.getOperand(OpIdx++));
7970b57cec5SDimitry Andric 
7980b57cec5SDimitry Andric   // Grab the super-register source.
7990b57cec5SDimitry Andric   MachineOperand MO = MI.getOperand(OpIdx++);
8000b57cec5SDimitry Andric   if (!TableEntry->IsLoad)
8010b57cec5SDimitry Andric     GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
8020b57cec5SDimitry Andric 
8030b57cec5SDimitry Andric   // Add the subregs as sources of the new instruction.
8040b57cec5SDimitry Andric   unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |
8050b57cec5SDimitry Andric                        getKillRegState(MO.isKill()));
8060b57cec5SDimitry Andric   MIB.addReg(D0, SrcFlags);
8070b57cec5SDimitry Andric   if (NumRegs > 1)
8080b57cec5SDimitry Andric     MIB.addReg(D1, SrcFlags);
8090b57cec5SDimitry Andric   if (NumRegs > 2)
8100b57cec5SDimitry Andric     MIB.addReg(D2, SrcFlags);
8110b57cec5SDimitry Andric   if (NumRegs > 3)
8120b57cec5SDimitry Andric     MIB.addReg(D3, SrcFlags);
8130b57cec5SDimitry Andric 
8140b57cec5SDimitry Andric   // Add the lane number operand.
8150b57cec5SDimitry Andric   MIB.addImm(Lane);
8160b57cec5SDimitry Andric   OpIdx += 1;
8170b57cec5SDimitry Andric 
8180b57cec5SDimitry Andric   // Copy the predicate operands.
8190b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
8200b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
8210b57cec5SDimitry Andric 
8220b57cec5SDimitry Andric   // Copy the super-register source to be an implicit source.
8230b57cec5SDimitry Andric   MO.setImplicit(true);
8240b57cec5SDimitry Andric   MIB.add(MO);
8250b57cec5SDimitry Andric   if (TableEntry->IsLoad)
8260b57cec5SDimitry Andric     // Add an implicit def for the super-register.
8270b57cec5SDimitry Andric     MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
82806c3fb27SDimitry Andric   MIB.copyImplicitOps(MI);
8290b57cec5SDimitry Andric   // Transfer memoperands.
8300b57cec5SDimitry Andric   MIB.cloneMemRefs(MI);
8310b57cec5SDimitry Andric   MI.eraseFromParent();
8320b57cec5SDimitry Andric }
8330b57cec5SDimitry Andric 
8340b57cec5SDimitry Andric /// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
8350b57cec5SDimitry Andric /// register operands to real instructions with D register operands.
8360b57cec5SDimitry Andric void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
8370b57cec5SDimitry Andric                                  unsigned Opc, bool IsExt) {
8380b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
8390b57cec5SDimitry Andric   MachineBasicBlock &MBB = *MI.getParent();
8400b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
8410b57cec5SDimitry Andric 
8420b57cec5SDimitry Andric   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));
8430b57cec5SDimitry Andric   unsigned OpIdx = 0;
8440b57cec5SDimitry Andric 
8450b57cec5SDimitry Andric   // Transfer the destination register operand.
8460b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
8470b57cec5SDimitry Andric   if (IsExt) {
8480b57cec5SDimitry Andric     MachineOperand VdSrc(MI.getOperand(OpIdx++));
8490b57cec5SDimitry Andric     MIB.add(VdSrc);
8500b57cec5SDimitry Andric   }
8510b57cec5SDimitry Andric 
8520b57cec5SDimitry Andric   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
8538bcb0991SDimitry Andric   Register SrcReg = MI.getOperand(OpIdx++).getReg();
8540b57cec5SDimitry Andric   unsigned D0, D1, D2, D3;
8550b57cec5SDimitry Andric   GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);
8560b57cec5SDimitry Andric   MIB.addReg(D0);
8570b57cec5SDimitry Andric 
8580b57cec5SDimitry Andric   // Copy the other source register operand.
8590b57cec5SDimitry Andric   MachineOperand VmSrc(MI.getOperand(OpIdx++));
8600b57cec5SDimitry Andric   MIB.add(VmSrc);
8610b57cec5SDimitry Andric 
8620b57cec5SDimitry Andric   // Copy the predicate operands.
8630b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
8640b57cec5SDimitry Andric   MIB.add(MI.getOperand(OpIdx++));
8650b57cec5SDimitry Andric 
8660b57cec5SDimitry Andric   // Add an implicit kill and use for the super-reg.
8670b57cec5SDimitry Andric   MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));
86806c3fb27SDimitry Andric   MIB.copyImplicitOps(MI);
8690b57cec5SDimitry Andric   MI.eraseFromParent();
8700b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "To:        "; MIB.getInstr()->dump(););
8710b57cec5SDimitry Andric }
8720b57cec5SDimitry Andric 
873349cc55cSDimitry Andric void ARMExpandPseudo::ExpandMQQPRLoadStore(MachineBasicBlock::iterator &MBBI) {
874349cc55cSDimitry Andric   MachineInstr &MI = *MBBI;
875349cc55cSDimitry Andric   MachineBasicBlock &MBB = *MI.getParent();
876349cc55cSDimitry Andric   unsigned NewOpc =
877349cc55cSDimitry Andric       MI.getOpcode() == ARM::MQQPRStore || MI.getOpcode() == ARM::MQQQQPRStore
878349cc55cSDimitry Andric           ? ARM::VSTMDIA
879349cc55cSDimitry Andric           : ARM::VLDMDIA;
880349cc55cSDimitry Andric   MachineInstrBuilder MIB =
881349cc55cSDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
882349cc55cSDimitry Andric 
883349cc55cSDimitry Andric   unsigned Flags = getKillRegState(MI.getOperand(0).isKill()) |
884349cc55cSDimitry Andric                    getDefRegState(MI.getOperand(0).isDef());
885349cc55cSDimitry Andric   Register SrcReg = MI.getOperand(0).getReg();
886349cc55cSDimitry Andric 
887349cc55cSDimitry Andric   // Copy the destination register.
888349cc55cSDimitry Andric   MIB.add(MI.getOperand(1));
889349cc55cSDimitry Andric   MIB.add(predOps(ARMCC::AL));
890349cc55cSDimitry Andric   MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_0), Flags);
891349cc55cSDimitry Andric   MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_1), Flags);
892349cc55cSDimitry Andric   MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_2), Flags);
893349cc55cSDimitry Andric   MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_3), Flags);
894349cc55cSDimitry Andric   if (MI.getOpcode() == ARM::MQQQQPRStore ||
895349cc55cSDimitry Andric       MI.getOpcode() == ARM::MQQQQPRLoad) {
896349cc55cSDimitry Andric     MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_4), Flags);
897349cc55cSDimitry Andric     MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_5), Flags);
898349cc55cSDimitry Andric     MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_6), Flags);
899349cc55cSDimitry Andric     MIB.addReg(TRI->getSubReg(SrcReg, ARM::dsub_7), Flags);
900349cc55cSDimitry Andric   }
901349cc55cSDimitry Andric 
902349cc55cSDimitry Andric   if (NewOpc == ARM::VSTMDIA)
903349cc55cSDimitry Andric     MIB.addReg(SrcReg, RegState::Implicit);
904349cc55cSDimitry Andric 
90506c3fb27SDimitry Andric   MIB.copyImplicitOps(MI);
906349cc55cSDimitry Andric   MIB.cloneMemRefs(MI);
907349cc55cSDimitry Andric   MI.eraseFromParent();
908349cc55cSDimitry Andric }
909349cc55cSDimitry Andric 
9100b57cec5SDimitry Andric static bool IsAnAddressOperand(const MachineOperand &MO) {
9110b57cec5SDimitry Andric   // This check is overly conservative.  Unless we are certain that the machine
9120b57cec5SDimitry Andric   // operand is not a symbol reference, we return that it is a symbol reference.
9130b57cec5SDimitry Andric   // This is important as the load pair may not be split up Windows.
9140b57cec5SDimitry Andric   switch (MO.getType()) {
9150b57cec5SDimitry Andric   case MachineOperand::MO_Register:
9160b57cec5SDimitry Andric   case MachineOperand::MO_Immediate:
9170b57cec5SDimitry Andric   case MachineOperand::MO_CImmediate:
9180b57cec5SDimitry Andric   case MachineOperand::MO_FPImmediate:
9198bcb0991SDimitry Andric   case MachineOperand::MO_ShuffleMask:
9200b57cec5SDimitry Andric     return false;
9210b57cec5SDimitry Andric   case MachineOperand::MO_MachineBasicBlock:
9220b57cec5SDimitry Andric     return true;
9230b57cec5SDimitry Andric   case MachineOperand::MO_FrameIndex:
9240b57cec5SDimitry Andric     return false;
9250b57cec5SDimitry Andric   case MachineOperand::MO_ConstantPoolIndex:
9260b57cec5SDimitry Andric   case MachineOperand::MO_TargetIndex:
9270b57cec5SDimitry Andric   case MachineOperand::MO_JumpTableIndex:
9280b57cec5SDimitry Andric   case MachineOperand::MO_ExternalSymbol:
9290b57cec5SDimitry Andric   case MachineOperand::MO_GlobalAddress:
9300b57cec5SDimitry Andric   case MachineOperand::MO_BlockAddress:
9310b57cec5SDimitry Andric     return true;
9320b57cec5SDimitry Andric   case MachineOperand::MO_RegisterMask:
9330b57cec5SDimitry Andric   case MachineOperand::MO_RegisterLiveOut:
9340b57cec5SDimitry Andric     return false;
9350b57cec5SDimitry Andric   case MachineOperand::MO_Metadata:
9360b57cec5SDimitry Andric   case MachineOperand::MO_MCSymbol:
9370b57cec5SDimitry Andric     return true;
938bdd1243dSDimitry Andric   case MachineOperand::MO_DbgInstrRef:
9390b57cec5SDimitry Andric   case MachineOperand::MO_CFIIndex:
9400b57cec5SDimitry Andric     return false;
9410b57cec5SDimitry Andric   case MachineOperand::MO_IntrinsicID:
9420b57cec5SDimitry Andric   case MachineOperand::MO_Predicate:
9430b57cec5SDimitry Andric     llvm_unreachable("should not exist post-isel");
9440b57cec5SDimitry Andric   }
9450b57cec5SDimitry Andric   llvm_unreachable("unhandled machine operand type");
9460b57cec5SDimitry Andric }
9470b57cec5SDimitry Andric 
9480b57cec5SDimitry Andric static MachineOperand makeImplicit(const MachineOperand &MO) {
9490b57cec5SDimitry Andric   MachineOperand NewMO = MO;
9500b57cec5SDimitry Andric   NewMO.setImplicit();
9510b57cec5SDimitry Andric   return NewMO;
9520b57cec5SDimitry Andric }
9530b57cec5SDimitry Andric 
95406c3fb27SDimitry Andric static MachineOperand getMovOperand(const MachineOperand &MO,
95506c3fb27SDimitry Andric                                     unsigned TargetFlag) {
95606c3fb27SDimitry Andric   unsigned TF = MO.getTargetFlags() | TargetFlag;
95706c3fb27SDimitry Andric   switch (MO.getType()) {
95806c3fb27SDimitry Andric   case MachineOperand::MO_Immediate: {
95906c3fb27SDimitry Andric     unsigned Imm = MO.getImm();
96006c3fb27SDimitry Andric     switch (TargetFlag) {
96106c3fb27SDimitry Andric     case ARMII::MO_HI_8_15:
96206c3fb27SDimitry Andric       Imm = (Imm >> 24) & 0xff;
96306c3fb27SDimitry Andric       break;
96406c3fb27SDimitry Andric     case ARMII::MO_HI_0_7:
96506c3fb27SDimitry Andric       Imm = (Imm >> 16) & 0xff;
96606c3fb27SDimitry Andric       break;
96706c3fb27SDimitry Andric     case ARMII::MO_LO_8_15:
96806c3fb27SDimitry Andric       Imm = (Imm >> 8) & 0xff;
96906c3fb27SDimitry Andric       break;
97006c3fb27SDimitry Andric     case ARMII::MO_LO_0_7:
97106c3fb27SDimitry Andric       Imm = Imm & 0xff;
97206c3fb27SDimitry Andric       break;
97306c3fb27SDimitry Andric     case ARMII::MO_HI16:
97406c3fb27SDimitry Andric       Imm = (Imm >> 16) & 0xffff;
97506c3fb27SDimitry Andric       break;
97606c3fb27SDimitry Andric     case ARMII::MO_LO16:
97706c3fb27SDimitry Andric       Imm = Imm & 0xffff;
97806c3fb27SDimitry Andric       break;
97906c3fb27SDimitry Andric     default:
98006c3fb27SDimitry Andric       llvm_unreachable("Only HI/LO target flags are expected");
98106c3fb27SDimitry Andric     }
98206c3fb27SDimitry Andric     return MachineOperand::CreateImm(Imm);
98306c3fb27SDimitry Andric   }
98406c3fb27SDimitry Andric   case MachineOperand::MO_ExternalSymbol:
98506c3fb27SDimitry Andric     return MachineOperand::CreateES(MO.getSymbolName(), TF);
98606c3fb27SDimitry Andric   case MachineOperand::MO_JumpTableIndex:
98706c3fb27SDimitry Andric     return MachineOperand::CreateJTI(MO.getIndex(), TF);
98806c3fb27SDimitry Andric   default:
98906c3fb27SDimitry Andric     return MachineOperand::CreateGA(MO.getGlobal(), MO.getOffset(), TF);
99006c3fb27SDimitry Andric   }
99106c3fb27SDimitry Andric }
99206c3fb27SDimitry Andric 
99306c3fb27SDimitry Andric void ARMExpandPseudo::ExpandTMOV32BitImm(MachineBasicBlock &MBB,
99406c3fb27SDimitry Andric                                          MachineBasicBlock::iterator &MBBI) {
99506c3fb27SDimitry Andric   MachineInstr &MI = *MBBI;
99606c3fb27SDimitry Andric   Register DstReg = MI.getOperand(0).getReg();
99706c3fb27SDimitry Andric   bool DstIsDead = MI.getOperand(0).isDead();
99806c3fb27SDimitry Andric   const MachineOperand &MO = MI.getOperand(1);
99906c3fb27SDimitry Andric   unsigned MIFlags = MI.getFlags();
100006c3fb27SDimitry Andric 
100106c3fb27SDimitry Andric   LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
100206c3fb27SDimitry Andric 
100306c3fb27SDimitry Andric   // Expand the mov into a sequence of mov/add+lsl of the individual bytes. We
100406c3fb27SDimitry Andric   // want to avoid emitting any zero bytes, as they won't change the result, and
100506c3fb27SDimitry Andric   // also don't want any pointless shifts, so instead of immediately emitting
100606c3fb27SDimitry Andric   // the shift for a byte we keep track of how much we will need to shift and do
100706c3fb27SDimitry Andric   // it before the next nonzero byte.
100806c3fb27SDimitry Andric   unsigned PendingShift = 0;
100906c3fb27SDimitry Andric   for (unsigned Byte = 0; Byte < 4; ++Byte) {
101006c3fb27SDimitry Andric     unsigned Flag = Byte == 0   ? ARMII::MO_HI_8_15
101106c3fb27SDimitry Andric                     : Byte == 1 ? ARMII::MO_HI_0_7
101206c3fb27SDimitry Andric                     : Byte == 2 ? ARMII::MO_LO_8_15
101306c3fb27SDimitry Andric                                 : ARMII::MO_LO_0_7;
101406c3fb27SDimitry Andric     MachineOperand Operand = getMovOperand(MO, Flag);
101506c3fb27SDimitry Andric     bool ZeroImm = Operand.isImm() && Operand.getImm() == 0;
101606c3fb27SDimitry Andric     unsigned Op = PendingShift ? ARM::tADDi8 : ARM::tMOVi8;
101706c3fb27SDimitry Andric 
101806c3fb27SDimitry Andric     // Emit the pending shift if we're going to emit this byte or if we've
101906c3fb27SDimitry Andric     // reached the end.
102006c3fb27SDimitry Andric     if (PendingShift && (!ZeroImm || Byte == 3)) {
102106c3fb27SDimitry Andric       MachineInstr *Lsl =
102206c3fb27SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tLSLri), DstReg)
102306c3fb27SDimitry Andric               .add(t1CondCodeOp(true))
102406c3fb27SDimitry Andric               .addReg(DstReg)
102506c3fb27SDimitry Andric               .addImm(PendingShift)
102606c3fb27SDimitry Andric               .add(predOps(ARMCC::AL))
102706c3fb27SDimitry Andric               .setMIFlags(MIFlags);
102806c3fb27SDimitry Andric       (void)Lsl;
102906c3fb27SDimitry Andric       LLVM_DEBUG(dbgs() << "And:       "; Lsl->dump(););
103006c3fb27SDimitry Andric       PendingShift = 0;
103106c3fb27SDimitry Andric     }
103206c3fb27SDimitry Andric 
103306c3fb27SDimitry Andric     // Emit this byte if it's nonzero.
103406c3fb27SDimitry Andric     if (!ZeroImm) {
103506c3fb27SDimitry Andric       MachineInstrBuilder MIB =
103606c3fb27SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Op), DstReg)
103706c3fb27SDimitry Andric               .add(t1CondCodeOp(true));
103806c3fb27SDimitry Andric       if (Op == ARM::tADDi8)
103906c3fb27SDimitry Andric         MIB.addReg(DstReg);
104006c3fb27SDimitry Andric       MIB.add(Operand);
104106c3fb27SDimitry Andric       MIB.add(predOps(ARMCC::AL));
104206c3fb27SDimitry Andric       MIB.setMIFlags(MIFlags);
104306c3fb27SDimitry Andric       LLVM_DEBUG(dbgs() << (Op == ARM::tMOVi8 ? "To: " : "And:") << "       ";
104406c3fb27SDimitry Andric                  MIB.getInstr()->dump(););
104506c3fb27SDimitry Andric     }
104606c3fb27SDimitry Andric 
104706c3fb27SDimitry Andric     // Don't accumulate the shift value if we've not yet seen a nonzero byte.
104806c3fb27SDimitry Andric     if (PendingShift || !ZeroImm)
104906c3fb27SDimitry Andric       PendingShift += 8;
105006c3fb27SDimitry Andric   }
105106c3fb27SDimitry Andric 
105206c3fb27SDimitry Andric   // The dest is dead on the last instruction we emitted if it was dead on the
105306c3fb27SDimitry Andric   // original instruction.
105406c3fb27SDimitry Andric   (--MBBI)->getOperand(0).setIsDead(DstIsDead);
105506c3fb27SDimitry Andric 
105606c3fb27SDimitry Andric   MI.eraseFromParent();
105706c3fb27SDimitry Andric }
105806c3fb27SDimitry Andric 
10590b57cec5SDimitry Andric void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
10600b57cec5SDimitry Andric                                         MachineBasicBlock::iterator &MBBI) {
10610b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
10620b57cec5SDimitry Andric   unsigned Opcode = MI.getOpcode();
10635ffd83dbSDimitry Andric   Register PredReg;
10640b57cec5SDimitry Andric   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
10658bcb0991SDimitry Andric   Register DstReg = MI.getOperand(0).getReg();
10660b57cec5SDimitry Andric   bool DstIsDead = MI.getOperand(0).isDead();
10670b57cec5SDimitry Andric   bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;
10680b57cec5SDimitry Andric   const MachineOperand &MO = MI.getOperand(isCC ? 2 : 1);
10690b57cec5SDimitry Andric   bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);
10700b57cec5SDimitry Andric   MachineInstrBuilder LO16, HI16;
10710b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
10720b57cec5SDimitry Andric 
10730b57cec5SDimitry Andric   if (!STI->hasV6T2Ops() &&
10740b57cec5SDimitry Andric       (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {
10750b57cec5SDimitry Andric     // FIXME Windows CE supports older ARM CPUs
10760b57cec5SDimitry Andric     assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
10770b57cec5SDimitry Andric 
1078e8d8bef9SDimitry Andric     assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
1079e8d8bef9SDimitry Andric     unsigned ImmVal = (unsigned)MO.getImm();
1080e8d8bef9SDimitry Andric     unsigned SOImmValV1 = 0, SOImmValV2 = 0;
1081e8d8bef9SDimitry Andric 
1082e8d8bef9SDimitry Andric     if (ARM_AM::isSOImmTwoPartVal(ImmVal)) { // Expand into a movi + orr.
10830b57cec5SDimitry Andric       LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
10840b57cec5SDimitry Andric       HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
10850b57cec5SDimitry Andric           .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
10860b57cec5SDimitry Andric           .addReg(DstReg);
1087e8d8bef9SDimitry Andric       SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
1088e8d8bef9SDimitry Andric       SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
1089e8d8bef9SDimitry Andric     } else { // Expand into a mvn + sub.
1090e8d8bef9SDimitry Andric       LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MVNi), DstReg);
1091e8d8bef9SDimitry Andric       HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri))
1092e8d8bef9SDimitry Andric           .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
1093e8d8bef9SDimitry Andric           .addReg(DstReg);
1094e8d8bef9SDimitry Andric       SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(-ImmVal);
1095e8d8bef9SDimitry Andric       SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(-ImmVal);
1096e8d8bef9SDimitry Andric       SOImmValV1 = ~(-SOImmValV1);
1097e8d8bef9SDimitry Andric     }
10980b57cec5SDimitry Andric 
10995ffd83dbSDimitry Andric     unsigned MIFlags = MI.getFlags();
11000b57cec5SDimitry Andric     LO16 = LO16.addImm(SOImmValV1);
11010b57cec5SDimitry Andric     HI16 = HI16.addImm(SOImmValV2);
11020b57cec5SDimitry Andric     LO16.cloneMemRefs(MI);
11030b57cec5SDimitry Andric     HI16.cloneMemRefs(MI);
11045ffd83dbSDimitry Andric     LO16.setMIFlags(MIFlags);
11055ffd83dbSDimitry Andric     HI16.setMIFlags(MIFlags);
11060b57cec5SDimitry Andric     LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());
11070b57cec5SDimitry Andric     HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());
11080b57cec5SDimitry Andric     if (isCC)
11090b57cec5SDimitry Andric       LO16.add(makeImplicit(MI.getOperand(1)));
111006c3fb27SDimitry Andric     LO16.copyImplicitOps(MI);
111106c3fb27SDimitry Andric     HI16.copyImplicitOps(MI);
11120b57cec5SDimitry Andric     MI.eraseFromParent();
11130b57cec5SDimitry Andric     return;
11140b57cec5SDimitry Andric   }
11150b57cec5SDimitry Andric 
11160b57cec5SDimitry Andric   unsigned LO16Opc = 0;
11170b57cec5SDimitry Andric   unsigned HI16Opc = 0;
11185ffd83dbSDimitry Andric   unsigned MIFlags = MI.getFlags();
11190b57cec5SDimitry Andric   if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
11200b57cec5SDimitry Andric     LO16Opc = ARM::t2MOVi16;
11210b57cec5SDimitry Andric     HI16Opc = ARM::t2MOVTi16;
11220b57cec5SDimitry Andric   } else {
11230b57cec5SDimitry Andric     LO16Opc = ARM::MOVi16;
11240b57cec5SDimitry Andric     HI16Opc = ARM::MOVTi16;
11250b57cec5SDimitry Andric   }
11260b57cec5SDimitry Andric 
11270b57cec5SDimitry Andric   LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
112806c3fb27SDimitry Andric   LO16.setMIFlags(MIFlags);
112906c3fb27SDimitry Andric   LO16.add(getMovOperand(MO, ARMII::MO_LO16));
113006c3fb27SDimitry Andric   LO16.cloneMemRefs(MI);
113106c3fb27SDimitry Andric   LO16.addImm(Pred).addReg(PredReg);
113206c3fb27SDimitry Andric   if (isCC)
113306c3fb27SDimitry Andric     LO16.add(makeImplicit(MI.getOperand(1)));
113406c3fb27SDimitry Andric   LO16.copyImplicitOps(MI);
113506c3fb27SDimitry Andric   LLVM_DEBUG(dbgs() << "To:        "; LO16.getInstr()->dump(););
113606c3fb27SDimitry Andric 
113706c3fb27SDimitry Andric   MachineOperand HIOperand = getMovOperand(MO, ARMII::MO_HI16);
113806c3fb27SDimitry Andric   if (!(HIOperand.isImm() && HIOperand.getImm() == 0)) {
11390b57cec5SDimitry Andric     HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
11400b57cec5SDimitry Andric                .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
11410b57cec5SDimitry Andric                .addReg(DstReg);
11425ffd83dbSDimitry Andric     HI16.setMIFlags(MIFlags);
114306c3fb27SDimitry Andric     HI16.add(HIOperand);
11440b57cec5SDimitry Andric     HI16.cloneMemRefs(MI);
11450b57cec5SDimitry Andric     HI16.addImm(Pred).addReg(PredReg);
114606c3fb27SDimitry Andric     HI16.copyImplicitOps(MI);
114706c3fb27SDimitry Andric     LLVM_DEBUG(dbgs() << "And:       "; HI16.getInstr()->dump(););
114806c3fb27SDimitry Andric   } else {
114906c3fb27SDimitry Andric     LO16->getOperand(0).setIsDead(DstIsDead);
115006c3fb27SDimitry Andric   }
11510b57cec5SDimitry Andric 
11520b57cec5SDimitry Andric   if (RequiresBundling)
11530b57cec5SDimitry Andric     finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());
11540b57cec5SDimitry Andric 
11550b57cec5SDimitry Andric   MI.eraseFromParent();
11560b57cec5SDimitry Andric }
11570b57cec5SDimitry Andric 
11585ffd83dbSDimitry Andric // The size of the area, accessed by that VLSTM/VLLDM
11595ffd83dbSDimitry Andric // S0-S31 + FPSCR + 8 more bytes (VPR + pad, or just pad)
11605ffd83dbSDimitry Andric static const int CMSE_FP_SAVE_SIZE = 136;
11615ffd83dbSDimitry Andric 
11625ffd83dbSDimitry Andric static void determineGPRegsToClear(const MachineInstr &MI,
11635ffd83dbSDimitry Andric                                    const std::initializer_list<unsigned> &Regs,
11645ffd83dbSDimitry Andric                                    SmallVectorImpl<unsigned> &ClearRegs) {
11655ffd83dbSDimitry Andric   SmallVector<unsigned, 4> OpRegs;
11665ffd83dbSDimitry Andric   for (const MachineOperand &Op : MI.operands()) {
11675ffd83dbSDimitry Andric     if (!Op.isReg() || !Op.isUse())
11685ffd83dbSDimitry Andric       continue;
11695ffd83dbSDimitry Andric     OpRegs.push_back(Op.getReg());
11705ffd83dbSDimitry Andric   }
11715ffd83dbSDimitry Andric   llvm::sort(OpRegs);
11725ffd83dbSDimitry Andric 
11735ffd83dbSDimitry Andric   std::set_difference(Regs.begin(), Regs.end(), OpRegs.begin(), OpRegs.end(),
11745ffd83dbSDimitry Andric                       std::back_inserter(ClearRegs));
11755ffd83dbSDimitry Andric }
11765ffd83dbSDimitry Andric 
11775ffd83dbSDimitry Andric void ARMExpandPseudo::CMSEClearGPRegs(
11785ffd83dbSDimitry Andric     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
11795ffd83dbSDimitry Andric     const DebugLoc &DL, const SmallVectorImpl<unsigned> &ClearRegs,
11805ffd83dbSDimitry Andric     unsigned ClobberReg) {
11815ffd83dbSDimitry Andric 
11825ffd83dbSDimitry Andric   if (STI->hasV8_1MMainlineOps()) {
11835ffd83dbSDimitry Andric     // Clear the registers using the CLRM instruction.
11845ffd83dbSDimitry Andric     MachineInstrBuilder CLRM =
11855ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::t2CLRM)).add(predOps(ARMCC::AL));
11865ffd83dbSDimitry Andric     for (unsigned R : ClearRegs)
11875ffd83dbSDimitry Andric       CLRM.addReg(R, RegState::Define);
11885ffd83dbSDimitry Andric     CLRM.addReg(ARM::APSR, RegState::Define);
11895ffd83dbSDimitry Andric     CLRM.addReg(ARM::CPSR, RegState::Define | RegState::Implicit);
11905ffd83dbSDimitry Andric   } else {
11915ffd83dbSDimitry Andric     // Clear the registers and flags by copying ClobberReg into them.
11925ffd83dbSDimitry Andric     // (Baseline can't do a high register clear in one instruction).
11935ffd83dbSDimitry Andric     for (unsigned Reg : ClearRegs) {
11945ffd83dbSDimitry Andric       if (Reg == ClobberReg)
11955ffd83dbSDimitry Andric         continue;
11965ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::tMOVr), Reg)
11975ffd83dbSDimitry Andric           .addReg(ClobberReg)
11985ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
11995ffd83dbSDimitry Andric     }
12005ffd83dbSDimitry Andric 
12015ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::t2MSR_M))
12025ffd83dbSDimitry Andric         .addImm(STI->hasDSP() ? 0xc00 : 0x800)
12035ffd83dbSDimitry Andric         .addReg(ClobberReg)
12045ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
12055ffd83dbSDimitry Andric   }
12065ffd83dbSDimitry Andric }
12075ffd83dbSDimitry Andric 
12085ffd83dbSDimitry Andric // Find which FP registers need to be cleared.  The parameter `ClearRegs` is
12095ffd83dbSDimitry Andric // initialised with all elements set to true, and this function resets all the
12105ffd83dbSDimitry Andric // bits, which correspond to register uses. Returns true if any floating point
12115ffd83dbSDimitry Andric // register is defined, false otherwise.
12125ffd83dbSDimitry Andric static bool determineFPRegsToClear(const MachineInstr &MI,
12135ffd83dbSDimitry Andric                                    BitVector &ClearRegs) {
12145ffd83dbSDimitry Andric   bool DefFP = false;
12155ffd83dbSDimitry Andric   for (const MachineOperand &Op : MI.operands()) {
12165ffd83dbSDimitry Andric     if (!Op.isReg())
12175ffd83dbSDimitry Andric       continue;
12185ffd83dbSDimitry Andric 
121904eeddc0SDimitry Andric     Register Reg = Op.getReg();
12205ffd83dbSDimitry Andric     if (Op.isDef()) {
12215ffd83dbSDimitry Andric       if ((Reg >= ARM::Q0 && Reg <= ARM::Q7) ||
12225ffd83dbSDimitry Andric           (Reg >= ARM::D0 && Reg <= ARM::D15) ||
12235ffd83dbSDimitry Andric           (Reg >= ARM::S0 && Reg <= ARM::S31))
12245ffd83dbSDimitry Andric         DefFP = true;
12255ffd83dbSDimitry Andric       continue;
12265ffd83dbSDimitry Andric     }
12275ffd83dbSDimitry Andric 
12285ffd83dbSDimitry Andric     if (Reg >= ARM::Q0 && Reg <= ARM::Q7) {
12295ffd83dbSDimitry Andric       int R = Reg - ARM::Q0;
12305ffd83dbSDimitry Andric       ClearRegs.reset(R * 4, (R + 1) * 4);
12315ffd83dbSDimitry Andric     } else if (Reg >= ARM::D0 && Reg <= ARM::D15) {
12325ffd83dbSDimitry Andric       int R = Reg - ARM::D0;
12335ffd83dbSDimitry Andric       ClearRegs.reset(R * 2, (R + 1) * 2);
12345ffd83dbSDimitry Andric     } else if (Reg >= ARM::S0 && Reg <= ARM::S31) {
12355ffd83dbSDimitry Andric       ClearRegs[Reg - ARM::S0] = false;
12365ffd83dbSDimitry Andric     }
12375ffd83dbSDimitry Andric   }
12385ffd83dbSDimitry Andric   return DefFP;
12395ffd83dbSDimitry Andric }
12405ffd83dbSDimitry Andric 
12415ffd83dbSDimitry Andric MachineBasicBlock &
12425ffd83dbSDimitry Andric ARMExpandPseudo::CMSEClearFPRegs(MachineBasicBlock &MBB,
12435ffd83dbSDimitry Andric                                  MachineBasicBlock::iterator MBBI) {
12445ffd83dbSDimitry Andric   BitVector ClearRegs(16, true);
12455ffd83dbSDimitry Andric   (void)determineFPRegsToClear(*MBBI, ClearRegs);
12465ffd83dbSDimitry Andric 
12475ffd83dbSDimitry Andric   if (STI->hasV8_1MMainlineOps())
12485ffd83dbSDimitry Andric     return CMSEClearFPRegsV81(MBB, MBBI, ClearRegs);
12495ffd83dbSDimitry Andric   else
12505ffd83dbSDimitry Andric     return CMSEClearFPRegsV8(MBB, MBBI, ClearRegs);
12515ffd83dbSDimitry Andric }
12525ffd83dbSDimitry Andric 
12535ffd83dbSDimitry Andric // Clear the FP registers for v8.0-M, by copying over the content
12545ffd83dbSDimitry Andric // of LR. Uses R12 as a scratch register.
12555ffd83dbSDimitry Andric MachineBasicBlock &
12565ffd83dbSDimitry Andric ARMExpandPseudo::CMSEClearFPRegsV8(MachineBasicBlock &MBB,
12575ffd83dbSDimitry Andric                                    MachineBasicBlock::iterator MBBI,
12585ffd83dbSDimitry Andric                                    const BitVector &ClearRegs) {
12595ffd83dbSDimitry Andric   if (!STI->hasFPRegs())
12605ffd83dbSDimitry Andric     return MBB;
12615ffd83dbSDimitry Andric 
12625ffd83dbSDimitry Andric   auto &RetI = *MBBI;
12635ffd83dbSDimitry Andric   const DebugLoc &DL = RetI.getDebugLoc();
12645ffd83dbSDimitry Andric 
12655ffd83dbSDimitry Andric   // If optimising for minimum size, clear FP registers unconditionally.
12665ffd83dbSDimitry Andric   // Otherwise, check the CONTROL.SFPA (Secure Floating-Point Active) bit and
12675ffd83dbSDimitry Andric   // don't clear them if they belong to the non-secure state.
12685ffd83dbSDimitry Andric   MachineBasicBlock *ClearBB, *DoneBB;
12695ffd83dbSDimitry Andric   if (STI->hasMinSize()) {
12705ffd83dbSDimitry Andric     ClearBB = DoneBB = &MBB;
12715ffd83dbSDimitry Andric   } else {
12725ffd83dbSDimitry Andric     MachineFunction *MF = MBB.getParent();
12735ffd83dbSDimitry Andric     ClearBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
12745ffd83dbSDimitry Andric     DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
12755ffd83dbSDimitry Andric 
12765ffd83dbSDimitry Andric     MF->insert(++MBB.getIterator(), ClearBB);
12775ffd83dbSDimitry Andric     MF->insert(++ClearBB->getIterator(), DoneBB);
12785ffd83dbSDimitry Andric 
12795ffd83dbSDimitry Andric     DoneBB->splice(DoneBB->end(), &MBB, MBBI, MBB.end());
12805ffd83dbSDimitry Andric     DoneBB->transferSuccessors(&MBB);
12815ffd83dbSDimitry Andric     MBB.addSuccessor(ClearBB);
12825ffd83dbSDimitry Andric     MBB.addSuccessor(DoneBB);
12835ffd83dbSDimitry Andric     ClearBB->addSuccessor(DoneBB);
12845ffd83dbSDimitry Andric 
12855ffd83dbSDimitry Andric     // At the new basic blocks we need to have live-in the registers, used
12865ffd83dbSDimitry Andric     // for the return value as well as LR, used to clear registers.
12875ffd83dbSDimitry Andric     for (const MachineOperand &Op : RetI.operands()) {
12885ffd83dbSDimitry Andric       if (!Op.isReg())
12895ffd83dbSDimitry Andric         continue;
12905ffd83dbSDimitry Andric       Register Reg = Op.getReg();
12915ffd83dbSDimitry Andric       if (Reg == ARM::NoRegister || Reg == ARM::LR)
12925ffd83dbSDimitry Andric         continue;
1293bdd1243dSDimitry Andric       assert(Reg.isPhysical() && "Unallocated register");
12945ffd83dbSDimitry Andric       ClearBB->addLiveIn(Reg);
12955ffd83dbSDimitry Andric       DoneBB->addLiveIn(Reg);
12965ffd83dbSDimitry Andric     }
12975ffd83dbSDimitry Andric     ClearBB->addLiveIn(ARM::LR);
12985ffd83dbSDimitry Andric     DoneBB->addLiveIn(ARM::LR);
12995ffd83dbSDimitry Andric 
13005ffd83dbSDimitry Andric     // Read the CONTROL register.
13015ffd83dbSDimitry Andric     BuildMI(MBB, MBB.end(), DL, TII->get(ARM::t2MRS_M), ARM::R12)
13025ffd83dbSDimitry Andric         .addImm(20)
13035ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
13045ffd83dbSDimitry Andric     // Check bit 3 (SFPA).
13055ffd83dbSDimitry Andric     BuildMI(MBB, MBB.end(), DL, TII->get(ARM::t2TSTri))
13065ffd83dbSDimitry Andric         .addReg(ARM::R12)
13075ffd83dbSDimitry Andric         .addImm(8)
13085ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
13095ffd83dbSDimitry Andric     // If SFPA is clear, jump over ClearBB to DoneBB.
13105ffd83dbSDimitry Andric     BuildMI(MBB, MBB.end(), DL, TII->get(ARM::tBcc))
13115ffd83dbSDimitry Andric         .addMBB(DoneBB)
13125ffd83dbSDimitry Andric         .addImm(ARMCC::EQ)
13135ffd83dbSDimitry Andric         .addReg(ARM::CPSR, RegState::Kill);
13145ffd83dbSDimitry Andric   }
13155ffd83dbSDimitry Andric 
13165ffd83dbSDimitry Andric   // Emit the clearing sequence
13175ffd83dbSDimitry Andric   for (unsigned D = 0; D < 8; D++) {
13185ffd83dbSDimitry Andric     // Attempt to clear as double
13195ffd83dbSDimitry Andric     if (ClearRegs[D * 2 + 0] && ClearRegs[D * 2 + 1]) {
13205ffd83dbSDimitry Andric       unsigned Reg = ARM::D0 + D;
13215ffd83dbSDimitry Andric       BuildMI(ClearBB, DL, TII->get(ARM::VMOVDRR), Reg)
13225ffd83dbSDimitry Andric           .addReg(ARM::LR)
13235ffd83dbSDimitry Andric           .addReg(ARM::LR)
13245ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
13255ffd83dbSDimitry Andric     } else {
13265ffd83dbSDimitry Andric       // Clear first part as single
13275ffd83dbSDimitry Andric       if (ClearRegs[D * 2 + 0]) {
13285ffd83dbSDimitry Andric         unsigned Reg = ARM::S0 + D * 2;
13295ffd83dbSDimitry Andric         BuildMI(ClearBB, DL, TII->get(ARM::VMOVSR), Reg)
13305ffd83dbSDimitry Andric             .addReg(ARM::LR)
13315ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
13325ffd83dbSDimitry Andric       }
13335ffd83dbSDimitry Andric       // Clear second part as single
13345ffd83dbSDimitry Andric       if (ClearRegs[D * 2 + 1]) {
13355ffd83dbSDimitry Andric         unsigned Reg = ARM::S0 + D * 2 + 1;
13365ffd83dbSDimitry Andric         BuildMI(ClearBB, DL, TII->get(ARM::VMOVSR), Reg)
13375ffd83dbSDimitry Andric             .addReg(ARM::LR)
13385ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
13395ffd83dbSDimitry Andric       }
13405ffd83dbSDimitry Andric     }
13415ffd83dbSDimitry Andric   }
13425ffd83dbSDimitry Andric 
13435ffd83dbSDimitry Andric   // Clear FPSCR bits 0-4, 7, 28-31
13445ffd83dbSDimitry Andric   // The other bits are program global according to the AAPCS
13455ffd83dbSDimitry Andric   BuildMI(ClearBB, DL, TII->get(ARM::VMRS), ARM::R12)
13465ffd83dbSDimitry Andric       .add(predOps(ARMCC::AL));
13475ffd83dbSDimitry Andric   BuildMI(ClearBB, DL, TII->get(ARM::t2BICri), ARM::R12)
13485ffd83dbSDimitry Andric       .addReg(ARM::R12)
13495ffd83dbSDimitry Andric       .addImm(0x0000009F)
13505ffd83dbSDimitry Andric       .add(predOps(ARMCC::AL))
13515ffd83dbSDimitry Andric       .add(condCodeOp());
13525ffd83dbSDimitry Andric   BuildMI(ClearBB, DL, TII->get(ARM::t2BICri), ARM::R12)
13535ffd83dbSDimitry Andric       .addReg(ARM::R12)
13545ffd83dbSDimitry Andric       .addImm(0xF0000000)
13555ffd83dbSDimitry Andric       .add(predOps(ARMCC::AL))
13565ffd83dbSDimitry Andric       .add(condCodeOp());
13575ffd83dbSDimitry Andric   BuildMI(ClearBB, DL, TII->get(ARM::VMSR))
13585ffd83dbSDimitry Andric       .addReg(ARM::R12)
13595ffd83dbSDimitry Andric       .add(predOps(ARMCC::AL));
13605ffd83dbSDimitry Andric 
13615ffd83dbSDimitry Andric   return *DoneBB;
13625ffd83dbSDimitry Andric }
13635ffd83dbSDimitry Andric 
13645ffd83dbSDimitry Andric MachineBasicBlock &
13655ffd83dbSDimitry Andric ARMExpandPseudo::CMSEClearFPRegsV81(MachineBasicBlock &MBB,
13665ffd83dbSDimitry Andric                                     MachineBasicBlock::iterator MBBI,
13675ffd83dbSDimitry Andric                                     const BitVector &ClearRegs) {
13685ffd83dbSDimitry Andric   auto &RetI = *MBBI;
13695ffd83dbSDimitry Andric 
13705ffd83dbSDimitry Andric   // Emit a sequence of VSCCLRM <sreglist> instructions, one instruction for
13715ffd83dbSDimitry Andric   // each contiguous sequence of S-registers.
13725ffd83dbSDimitry Andric   int Start = -1, End = -1;
13735ffd83dbSDimitry Andric   for (int S = 0, E = ClearRegs.size(); S != E; ++S) {
13745ffd83dbSDimitry Andric     if (ClearRegs[S] && S == End + 1) {
13755ffd83dbSDimitry Andric       End = S; // extend range
13765ffd83dbSDimitry Andric       continue;
13775ffd83dbSDimitry Andric     }
13785ffd83dbSDimitry Andric     // Emit current range.
13795ffd83dbSDimitry Andric     if (Start < End) {
13805ffd83dbSDimitry Andric       MachineInstrBuilder VSCCLRM =
13815ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, RetI.getDebugLoc(), TII->get(ARM::VSCCLRMS))
13825ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL));
13835ffd83dbSDimitry Andric       while (++Start <= End)
13845ffd83dbSDimitry Andric         VSCCLRM.addReg(ARM::S0 + Start, RegState::Define);
13855ffd83dbSDimitry Andric       VSCCLRM.addReg(ARM::VPR, RegState::Define);
13865ffd83dbSDimitry Andric     }
13875ffd83dbSDimitry Andric     Start = End = S;
13885ffd83dbSDimitry Andric   }
13895ffd83dbSDimitry Andric   // Emit last range.
13905ffd83dbSDimitry Andric   if (Start < End) {
13915ffd83dbSDimitry Andric     MachineInstrBuilder VSCCLRM =
13925ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, RetI.getDebugLoc(), TII->get(ARM::VSCCLRMS))
13935ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
13945ffd83dbSDimitry Andric     while (++Start <= End)
13955ffd83dbSDimitry Andric       VSCCLRM.addReg(ARM::S0 + Start, RegState::Define);
13965ffd83dbSDimitry Andric     VSCCLRM.addReg(ARM::VPR, RegState::Define);
13975ffd83dbSDimitry Andric   }
13985ffd83dbSDimitry Andric 
13995ffd83dbSDimitry Andric   return MBB;
14005ffd83dbSDimitry Andric }
14015ffd83dbSDimitry Andric 
14025ffd83dbSDimitry Andric void ARMExpandPseudo::CMSESaveClearFPRegs(
14035ffd83dbSDimitry Andric     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
14045ffd83dbSDimitry Andric     const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {
14055ffd83dbSDimitry Andric   if (STI->hasV8_1MMainlineOps())
14065ffd83dbSDimitry Andric     CMSESaveClearFPRegsV81(MBB, MBBI, DL, LiveRegs);
1407349cc55cSDimitry Andric   else if (STI->hasV8MMainlineOps())
14085ffd83dbSDimitry Andric     CMSESaveClearFPRegsV8(MBB, MBBI, DL, LiveRegs, ScratchRegs);
14095ffd83dbSDimitry Andric }
14105ffd83dbSDimitry Andric 
14115ffd83dbSDimitry Andric // Save and clear FP registers if present
14125ffd83dbSDimitry Andric void ARMExpandPseudo::CMSESaveClearFPRegsV8(
14135ffd83dbSDimitry Andric     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
14145ffd83dbSDimitry Andric     const LivePhysRegs &LiveRegs, SmallVectorImpl<unsigned> &ScratchRegs) {
14155ffd83dbSDimitry Andric 
14165ffd83dbSDimitry Andric   // Store an available register for FPSCR clearing
14175ffd83dbSDimitry Andric   assert(!ScratchRegs.empty());
14185ffd83dbSDimitry Andric   unsigned SpareReg = ScratchRegs.front();
14195ffd83dbSDimitry Andric 
14205ffd83dbSDimitry Andric   // save space on stack for VLSTM
14215ffd83dbSDimitry Andric   BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBspi), ARM::SP)
14225ffd83dbSDimitry Andric       .addReg(ARM::SP)
14235ffd83dbSDimitry Andric       .addImm(CMSE_FP_SAVE_SIZE >> 2)
14245ffd83dbSDimitry Andric       .add(predOps(ARMCC::AL));
14255ffd83dbSDimitry Andric 
14265ffd83dbSDimitry Andric   // Use ScratchRegs to store the fp regs
14275ffd83dbSDimitry Andric   std::vector<std::tuple<unsigned, unsigned, unsigned>> ClearedFPRegs;
14285ffd83dbSDimitry Andric   std::vector<unsigned> NonclearedFPRegs;
14295ffd83dbSDimitry Andric   for (const MachineOperand &Op : MBBI->operands()) {
14305ffd83dbSDimitry Andric     if (Op.isReg() && Op.isUse()) {
143104eeddc0SDimitry Andric       Register Reg = Op.getReg();
14325ffd83dbSDimitry Andric       assert(!ARM::DPRRegClass.contains(Reg) ||
14335ffd83dbSDimitry Andric              ARM::DPR_VFP2RegClass.contains(Reg));
14345ffd83dbSDimitry Andric       assert(!ARM::QPRRegClass.contains(Reg));
14355ffd83dbSDimitry Andric       if (ARM::DPR_VFP2RegClass.contains(Reg)) {
14365ffd83dbSDimitry Andric         if (ScratchRegs.size() >= 2) {
14375ffd83dbSDimitry Andric           unsigned SaveReg2 = ScratchRegs.pop_back_val();
14385ffd83dbSDimitry Andric           unsigned SaveReg1 = ScratchRegs.pop_back_val();
14395ffd83dbSDimitry Andric           ClearedFPRegs.emplace_back(Reg, SaveReg1, SaveReg2);
14405ffd83dbSDimitry Andric 
14415ffd83dbSDimitry Andric           // Save the fp register to the normal registers
14425ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRRD))
14435ffd83dbSDimitry Andric               .addReg(SaveReg1, RegState::Define)
14445ffd83dbSDimitry Andric               .addReg(SaveReg2, RegState::Define)
14455ffd83dbSDimitry Andric               .addReg(Reg)
14465ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL));
14475ffd83dbSDimitry Andric         } else {
14485ffd83dbSDimitry Andric           NonclearedFPRegs.push_back(Reg);
14495ffd83dbSDimitry Andric         }
14505ffd83dbSDimitry Andric       } else if (ARM::SPRRegClass.contains(Reg)) {
14515ffd83dbSDimitry Andric         if (ScratchRegs.size() >= 1) {
14525ffd83dbSDimitry Andric           unsigned SaveReg = ScratchRegs.pop_back_val();
14535ffd83dbSDimitry Andric           ClearedFPRegs.emplace_back(Reg, SaveReg, 0);
14545ffd83dbSDimitry Andric 
14555ffd83dbSDimitry Andric           // Save the fp register to the normal registers
14565ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRS), SaveReg)
14575ffd83dbSDimitry Andric               .addReg(Reg)
14585ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL));
14595ffd83dbSDimitry Andric         } else {
14605ffd83dbSDimitry Andric           NonclearedFPRegs.push_back(Reg);
14615ffd83dbSDimitry Andric         }
14625ffd83dbSDimitry Andric       }
14635ffd83dbSDimitry Andric     }
14645ffd83dbSDimitry Andric   }
14655ffd83dbSDimitry Andric 
14665ffd83dbSDimitry Andric   bool passesFPReg = (!NonclearedFPRegs.empty() || !ClearedFPRegs.empty());
14675ffd83dbSDimitry Andric 
1468349cc55cSDimitry Andric   if (passesFPReg)
1469349cc55cSDimitry Andric     assert(STI->hasFPRegs() && "Subtarget needs fpregs");
1470349cc55cSDimitry Andric 
1471349cc55cSDimitry Andric   // Lazy store all fp registers to the stack.
1472349cc55cSDimitry Andric   // This executes as NOP in the absence of floating-point support.
1473*0fca6ea1SDimitry Andric   MachineInstrBuilder VLSTM =
1474*0fca6ea1SDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))
14755ffd83dbSDimitry Andric           .addReg(ARM::SP)
1476*0fca6ea1SDimitry Andric           .add(predOps(ARMCC::AL))
1477*0fca6ea1SDimitry Andric           .addImm(0); // Represents a pseoudo register list, has no effect on
1478*0fca6ea1SDimitry Andric                       // the encoding.
1479*0fca6ea1SDimitry Andric   // Mark non-live registers as undef
1480*0fca6ea1SDimitry Andric   for (MachineOperand &MO : VLSTM->implicit_operands()) {
1481*0fca6ea1SDimitry Andric     if (MO.isReg() && !MO.isDef()) {
1482*0fca6ea1SDimitry Andric       Register Reg = MO.getReg();
1483*0fca6ea1SDimitry Andric       MO.setIsUndef(!LiveRegs.contains(Reg));
1484*0fca6ea1SDimitry Andric     }
1485*0fca6ea1SDimitry Andric   }
14865ffd83dbSDimitry Andric 
14875ffd83dbSDimitry Andric   // Restore all arguments
14885ffd83dbSDimitry Andric   for (const auto &Regs : ClearedFPRegs) {
14895ffd83dbSDimitry Andric     unsigned Reg, SaveReg1, SaveReg2;
14905ffd83dbSDimitry Andric     std::tie(Reg, SaveReg1, SaveReg2) = Regs;
14915ffd83dbSDimitry Andric     if (ARM::DPR_VFP2RegClass.contains(Reg))
14925ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVDRR), Reg)
14935ffd83dbSDimitry Andric           .addReg(SaveReg1)
14945ffd83dbSDimitry Andric           .addReg(SaveReg2)
14955ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
14965ffd83dbSDimitry Andric     else if (ARM::SPRRegClass.contains(Reg))
14975ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVSR), Reg)
14985ffd83dbSDimitry Andric           .addReg(SaveReg1)
14995ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
15005ffd83dbSDimitry Andric   }
15015ffd83dbSDimitry Andric 
15025ffd83dbSDimitry Andric   for (unsigned Reg : NonclearedFPRegs) {
15035ffd83dbSDimitry Andric     if (ARM::DPR_VFP2RegClass.contains(Reg)) {
15045ffd83dbSDimitry Andric       if (STI->isLittle()) {
15055ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRD), Reg)
15065ffd83dbSDimitry Andric             .addReg(ARM::SP)
15075ffd83dbSDimitry Andric             .addImm((Reg - ARM::D0) * 2)
15085ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
15095ffd83dbSDimitry Andric       } else {
15105ffd83dbSDimitry Andric         // For big-endian targets we need to load the two subregisters of Reg
15115ffd83dbSDimitry Andric         // manually because VLDRD would load them in wrong order
15125ffd83dbSDimitry Andric         unsigned SReg0 = TRI->getSubReg(Reg, ARM::ssub_0);
15135ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRS), SReg0)
15145ffd83dbSDimitry Andric             .addReg(ARM::SP)
15155ffd83dbSDimitry Andric             .addImm((Reg - ARM::D0) * 2)
15165ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
15175ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRS), SReg0 + 1)
15185ffd83dbSDimitry Andric             .addReg(ARM::SP)
15195ffd83dbSDimitry Andric             .addImm((Reg - ARM::D0) * 2 + 1)
15205ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
15215ffd83dbSDimitry Andric       }
15225ffd83dbSDimitry Andric     } else if (ARM::SPRRegClass.contains(Reg)) {
15235ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDRS), Reg)
15245ffd83dbSDimitry Andric           .addReg(ARM::SP)
15255ffd83dbSDimitry Andric           .addImm(Reg - ARM::S0)
15265ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
15275ffd83dbSDimitry Andric     }
15285ffd83dbSDimitry Andric   }
15295ffd83dbSDimitry Andric   // restore FPSCR from stack and clear bits 0-4, 7, 28-31
15305ffd83dbSDimitry Andric   // The other bits are program global according to the AAPCS
15315ffd83dbSDimitry Andric   if (passesFPReg) {
153204eeddc0SDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::tLDRspi), SpareReg)
15335ffd83dbSDimitry Andric         .addReg(ARM::SP)
153404eeddc0SDimitry Andric         .addImm(0x10)
15355ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
15365ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::t2BICri), SpareReg)
15375ffd83dbSDimitry Andric         .addReg(SpareReg)
15385ffd83dbSDimitry Andric         .addImm(0x0000009F)
15395ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL))
15405ffd83dbSDimitry Andric         .add(condCodeOp());
15415ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::t2BICri), SpareReg)
15425ffd83dbSDimitry Andric         .addReg(SpareReg)
15435ffd83dbSDimitry Andric         .addImm(0xF0000000)
15445ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL))
15455ffd83dbSDimitry Andric         .add(condCodeOp());
15465ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::VMSR))
15475ffd83dbSDimitry Andric         .addReg(SpareReg)
15485ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
15495ffd83dbSDimitry Andric     // The ldr must happen after a floating point instruction. To prevent the
15505ffd83dbSDimitry Andric     // post-ra scheduler to mess with the order, we create a bundle.
15515ffd83dbSDimitry Andric     finalizeBundle(MBB, VLSTM->getIterator(), MBBI->getIterator());
15525ffd83dbSDimitry Andric   }
15535ffd83dbSDimitry Andric }
15545ffd83dbSDimitry Andric 
15555ffd83dbSDimitry Andric void ARMExpandPseudo::CMSESaveClearFPRegsV81(MachineBasicBlock &MBB,
15565ffd83dbSDimitry Andric                                              MachineBasicBlock::iterator MBBI,
15575ffd83dbSDimitry Andric                                              DebugLoc &DL,
15585ffd83dbSDimitry Andric                                              const LivePhysRegs &LiveRegs) {
15595ffd83dbSDimitry Andric   BitVector ClearRegs(32, true);
15605ffd83dbSDimitry Andric   bool DefFP = determineFPRegsToClear(*MBBI, ClearRegs);
15615ffd83dbSDimitry Andric 
15625ffd83dbSDimitry Andric   // If the instruction does not write to a FP register and no elements were
15635ffd83dbSDimitry Andric   // removed from the set, then no FP registers were used to pass
15645ffd83dbSDimitry Andric   // arguments/returns.
15655ffd83dbSDimitry Andric   if (!DefFP && ClearRegs.count() == ClearRegs.size()) {
15665ffd83dbSDimitry Andric     // save space on stack for VLSTM
15675ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBspi), ARM::SP)
15685ffd83dbSDimitry Andric         .addReg(ARM::SP)
15695ffd83dbSDimitry Andric         .addImm(CMSE_FP_SAVE_SIZE >> 2)
15705ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
15715ffd83dbSDimitry Andric 
15725ffd83dbSDimitry Andric     // Lazy store all FP registers to the stack
1573*0fca6ea1SDimitry Andric     MachineInstrBuilder VLSTM =
1574*0fca6ea1SDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::VLSTM))
15755ffd83dbSDimitry Andric             .addReg(ARM::SP)
1576*0fca6ea1SDimitry Andric             .add(predOps(ARMCC::AL))
1577*0fca6ea1SDimitry Andric             .addImm(0); // Represents a pseoudo register list, has no effect on
1578*0fca6ea1SDimitry Andric                         // the encoding.
1579*0fca6ea1SDimitry Andric     // Mark non-live registers as undef
1580*0fca6ea1SDimitry Andric     for (MachineOperand &MO : VLSTM->implicit_operands()) {
1581*0fca6ea1SDimitry Andric       if (MO.isReg() && MO.isImplicit() && !MO.isDef()) {
1582*0fca6ea1SDimitry Andric         Register Reg = MO.getReg();
1583*0fca6ea1SDimitry Andric         MO.setIsUndef(!LiveRegs.contains(Reg));
1584*0fca6ea1SDimitry Andric       }
1585*0fca6ea1SDimitry Andric     }
15865ffd83dbSDimitry Andric   } else {
15875ffd83dbSDimitry Andric     // Push all the callee-saved registers (s16-s31).
15885ffd83dbSDimitry Andric     MachineInstrBuilder VPUSH =
15895ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTMSDB_UPD), ARM::SP)
15905ffd83dbSDimitry Andric             .addReg(ARM::SP)
15915ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
15925ffd83dbSDimitry Andric     for (int Reg = ARM::S16; Reg <= ARM::S31; ++Reg)
15935ffd83dbSDimitry Andric       VPUSH.addReg(Reg);
15945ffd83dbSDimitry Andric 
15955ffd83dbSDimitry Andric     // Clear FP registers with a VSCCLRM.
15965ffd83dbSDimitry Andric     (void)CMSEClearFPRegsV81(MBB, MBBI, ClearRegs);
15975ffd83dbSDimitry Andric 
15985ffd83dbSDimitry Andric     // Save floating-point context.
15995ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTR_FPCXTS_pre), ARM::SP)
16005ffd83dbSDimitry Andric         .addReg(ARM::SP)
16015ffd83dbSDimitry Andric         .addImm(-8)
16025ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
16035ffd83dbSDimitry Andric   }
16045ffd83dbSDimitry Andric }
16055ffd83dbSDimitry Andric 
16065ffd83dbSDimitry Andric // Restore FP registers if present
16075ffd83dbSDimitry Andric void ARMExpandPseudo::CMSERestoreFPRegs(
16085ffd83dbSDimitry Andric     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
16095ffd83dbSDimitry Andric     SmallVectorImpl<unsigned> &AvailableRegs) {
16105ffd83dbSDimitry Andric   if (STI->hasV8_1MMainlineOps())
16115ffd83dbSDimitry Andric     CMSERestoreFPRegsV81(MBB, MBBI, DL, AvailableRegs);
1612349cc55cSDimitry Andric   else if (STI->hasV8MMainlineOps())
16135ffd83dbSDimitry Andric     CMSERestoreFPRegsV8(MBB, MBBI, DL, AvailableRegs);
16145ffd83dbSDimitry Andric }
16155ffd83dbSDimitry Andric 
16165ffd83dbSDimitry Andric void ARMExpandPseudo::CMSERestoreFPRegsV8(
16175ffd83dbSDimitry Andric     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
16185ffd83dbSDimitry Andric     SmallVectorImpl<unsigned> &AvailableRegs) {
1619349cc55cSDimitry Andric 
1620349cc55cSDimitry Andric   // Keep a scratch register for the mitigation sequence.
1621349cc55cSDimitry Andric   unsigned ScratchReg = ARM::NoRegister;
1622349cc55cSDimitry Andric   if (STI->fixCMSE_CVE_2021_35465())
1623349cc55cSDimitry Andric     ScratchReg = AvailableRegs.pop_back_val();
16245ffd83dbSDimitry Andric 
16255ffd83dbSDimitry Andric   // Use AvailableRegs to store the fp regs
16265ffd83dbSDimitry Andric   std::vector<std::tuple<unsigned, unsigned, unsigned>> ClearedFPRegs;
16275ffd83dbSDimitry Andric   std::vector<unsigned> NonclearedFPRegs;
16285ffd83dbSDimitry Andric   for (const MachineOperand &Op : MBBI->operands()) {
16295ffd83dbSDimitry Andric     if (Op.isReg() && Op.isDef()) {
163004eeddc0SDimitry Andric       Register Reg = Op.getReg();
16315ffd83dbSDimitry Andric       assert(!ARM::DPRRegClass.contains(Reg) ||
16325ffd83dbSDimitry Andric              ARM::DPR_VFP2RegClass.contains(Reg));
16335ffd83dbSDimitry Andric       assert(!ARM::QPRRegClass.contains(Reg));
16345ffd83dbSDimitry Andric       if (ARM::DPR_VFP2RegClass.contains(Reg)) {
16355ffd83dbSDimitry Andric         if (AvailableRegs.size() >= 2) {
16365ffd83dbSDimitry Andric           unsigned SaveReg2 = AvailableRegs.pop_back_val();
16375ffd83dbSDimitry Andric           unsigned SaveReg1 = AvailableRegs.pop_back_val();
16385ffd83dbSDimitry Andric           ClearedFPRegs.emplace_back(Reg, SaveReg1, SaveReg2);
16395ffd83dbSDimitry Andric 
16405ffd83dbSDimitry Andric           // Save the fp register to the normal registers
16415ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRRD))
16425ffd83dbSDimitry Andric               .addReg(SaveReg1, RegState::Define)
16435ffd83dbSDimitry Andric               .addReg(SaveReg2, RegState::Define)
16445ffd83dbSDimitry Andric               .addReg(Reg)
16455ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL));
16465ffd83dbSDimitry Andric         } else {
16475ffd83dbSDimitry Andric           NonclearedFPRegs.push_back(Reg);
16485ffd83dbSDimitry Andric         }
16495ffd83dbSDimitry Andric       } else if (ARM::SPRRegClass.contains(Reg)) {
16505ffd83dbSDimitry Andric         if (AvailableRegs.size() >= 1) {
16515ffd83dbSDimitry Andric           unsigned SaveReg = AvailableRegs.pop_back_val();
16525ffd83dbSDimitry Andric           ClearedFPRegs.emplace_back(Reg, SaveReg, 0);
16535ffd83dbSDimitry Andric 
16545ffd83dbSDimitry Andric           // Save the fp register to the normal registers
16555ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVRS), SaveReg)
16565ffd83dbSDimitry Andric               .addReg(Reg)
16575ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL));
16585ffd83dbSDimitry Andric         } else {
16595ffd83dbSDimitry Andric           NonclearedFPRegs.push_back(Reg);
16605ffd83dbSDimitry Andric         }
16615ffd83dbSDimitry Andric       }
16625ffd83dbSDimitry Andric     }
16635ffd83dbSDimitry Andric   }
16645ffd83dbSDimitry Andric 
1665349cc55cSDimitry Andric   bool returnsFPReg = (!NonclearedFPRegs.empty() || !ClearedFPRegs.empty());
1666349cc55cSDimitry Andric 
1667349cc55cSDimitry Andric   if (returnsFPReg)
1668349cc55cSDimitry Andric     assert(STI->hasFPRegs() && "Subtarget needs fpregs");
1669349cc55cSDimitry Andric 
16705ffd83dbSDimitry Andric   // Push FP regs that cannot be restored via normal registers on the stack
16715ffd83dbSDimitry Andric   for (unsigned Reg : NonclearedFPRegs) {
16725ffd83dbSDimitry Andric     if (ARM::DPR_VFP2RegClass.contains(Reg))
1673349cc55cSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTRD))
1674349cc55cSDimitry Andric           .addReg(Reg)
16755ffd83dbSDimitry Andric           .addReg(ARM::SP)
16765ffd83dbSDimitry Andric           .addImm((Reg - ARM::D0) * 2)
16775ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
16785ffd83dbSDimitry Andric     else if (ARM::SPRRegClass.contains(Reg))
1679349cc55cSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VSTRS))
1680349cc55cSDimitry Andric           .addReg(Reg)
16815ffd83dbSDimitry Andric           .addReg(ARM::SP)
16825ffd83dbSDimitry Andric           .addImm(Reg - ARM::S0)
16835ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
16845ffd83dbSDimitry Andric   }
16855ffd83dbSDimitry Andric 
1686349cc55cSDimitry Andric   // Lazy load fp regs from stack.
1687349cc55cSDimitry Andric   // This executes as NOP in the absence of floating-point support.
1688*0fca6ea1SDimitry Andric   MachineInstrBuilder VLLDM =
1689*0fca6ea1SDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM))
16905ffd83dbSDimitry Andric           .addReg(ARM::SP)
1691*0fca6ea1SDimitry Andric           .add(predOps(ARMCC::AL))
1692*0fca6ea1SDimitry Andric           .addImm(0); // Represents a pseoudo register list, has no effect on
1693*0fca6ea1SDimitry Andric                       // the encoding.
16945ffd83dbSDimitry Andric 
1695349cc55cSDimitry Andric   if (STI->fixCMSE_CVE_2021_35465()) {
1696349cc55cSDimitry Andric     auto Bundler = MIBundleBuilder(MBB, VLLDM);
1697349cc55cSDimitry Andric     // Read the CONTROL register.
1698349cc55cSDimitry Andric     Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::t2MRS_M))
1699349cc55cSDimitry Andric                        .addReg(ScratchReg, RegState::Define)
1700349cc55cSDimitry Andric                        .addImm(20)
1701349cc55cSDimitry Andric                        .add(predOps(ARMCC::AL)));
1702349cc55cSDimitry Andric     // Check bit 3 (SFPA).
1703349cc55cSDimitry Andric     Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::t2TSTri))
1704349cc55cSDimitry Andric                        .addReg(ScratchReg)
1705349cc55cSDimitry Andric                        .addImm(8)
1706349cc55cSDimitry Andric                        .add(predOps(ARMCC::AL)));
1707349cc55cSDimitry Andric     // Emit the IT block.
1708349cc55cSDimitry Andric     Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::t2IT))
1709349cc55cSDimitry Andric                        .addImm(ARMCC::NE)
1710349cc55cSDimitry Andric                        .addImm(8));
1711349cc55cSDimitry Andric     // If SFPA is clear jump over to VLLDM, otherwise execute an instruction
1712349cc55cSDimitry Andric     // which has no functional effect apart from causing context creation:
1713349cc55cSDimitry Andric     // vmovne s0, s0. In the absence of FPU we emit .inst.w 0xeeb00a40,
1714349cc55cSDimitry Andric     // which is defined as NOP if not executed.
1715349cc55cSDimitry Andric     if (STI->hasFPRegs())
1716349cc55cSDimitry Andric       Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::VMOVS))
1717349cc55cSDimitry Andric                          .addReg(ARM::S0, RegState::Define)
1718349cc55cSDimitry Andric                          .addReg(ARM::S0, RegState::Undef)
1719349cc55cSDimitry Andric                          .add(predOps(ARMCC::NE)));
1720349cc55cSDimitry Andric     else
1721349cc55cSDimitry Andric       Bundler.append(BuildMI(*MBB.getParent(), DL, TII->get(ARM::INLINEASM))
1722349cc55cSDimitry Andric                          .addExternalSymbol(".inst.w 0xeeb00a40")
1723349cc55cSDimitry Andric                          .addImm(InlineAsm::Extra_HasSideEffects));
1724349cc55cSDimitry Andric     finalizeBundle(MBB, Bundler.begin(), Bundler.end());
1725349cc55cSDimitry Andric   }
1726349cc55cSDimitry Andric 
17275ffd83dbSDimitry Andric   // Restore all FP registers via normal registers
17285ffd83dbSDimitry Andric   for (const auto &Regs : ClearedFPRegs) {
17295ffd83dbSDimitry Andric     unsigned Reg, SaveReg1, SaveReg2;
17305ffd83dbSDimitry Andric     std::tie(Reg, SaveReg1, SaveReg2) = Regs;
17315ffd83dbSDimitry Andric     if (ARM::DPR_VFP2RegClass.contains(Reg))
17325ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVDRR), Reg)
17335ffd83dbSDimitry Andric           .addReg(SaveReg1)
17345ffd83dbSDimitry Andric           .addReg(SaveReg2)
17355ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
17365ffd83dbSDimitry Andric     else if (ARM::SPRRegClass.contains(Reg))
17375ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VMOVSR), Reg)
17385ffd83dbSDimitry Andric           .addReg(SaveReg1)
17395ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
17405ffd83dbSDimitry Andric   }
17415ffd83dbSDimitry Andric 
17425ffd83dbSDimitry Andric   // Pop the stack space
17435ffd83dbSDimitry Andric   BuildMI(MBB, MBBI, DL, TII->get(ARM::tADDspi), ARM::SP)
17445ffd83dbSDimitry Andric       .addReg(ARM::SP)
17455ffd83dbSDimitry Andric       .addImm(CMSE_FP_SAVE_SIZE >> 2)
17465ffd83dbSDimitry Andric       .add(predOps(ARMCC::AL));
17475ffd83dbSDimitry Andric }
17485ffd83dbSDimitry Andric 
17495ffd83dbSDimitry Andric static bool definesOrUsesFPReg(const MachineInstr &MI) {
17505ffd83dbSDimitry Andric   for (const MachineOperand &Op : MI.operands()) {
17515ffd83dbSDimitry Andric     if (!Op.isReg())
17525ffd83dbSDimitry Andric       continue;
175304eeddc0SDimitry Andric     Register Reg = Op.getReg();
17545ffd83dbSDimitry Andric     if ((Reg >= ARM::Q0 && Reg <= ARM::Q7) ||
17555ffd83dbSDimitry Andric         (Reg >= ARM::D0 && Reg <= ARM::D15) ||
17565ffd83dbSDimitry Andric         (Reg >= ARM::S0 && Reg <= ARM::S31))
17575ffd83dbSDimitry Andric       return true;
17585ffd83dbSDimitry Andric   }
17595ffd83dbSDimitry Andric   return false;
17605ffd83dbSDimitry Andric }
17615ffd83dbSDimitry Andric 
17625ffd83dbSDimitry Andric void ARMExpandPseudo::CMSERestoreFPRegsV81(
17635ffd83dbSDimitry Andric     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc &DL,
17645ffd83dbSDimitry Andric     SmallVectorImpl<unsigned> &AvailableRegs) {
17655ffd83dbSDimitry Andric   if (!definesOrUsesFPReg(*MBBI)) {
1766349cc55cSDimitry Andric     if (STI->fixCMSE_CVE_2021_35465()) {
1767349cc55cSDimitry Andric       BuildMI(MBB, MBBI, DL, TII->get(ARM::VSCCLRMS))
1768349cc55cSDimitry Andric           .add(predOps(ARMCC::AL))
1769349cc55cSDimitry Andric           .addReg(ARM::VPR, RegState::Define);
1770349cc55cSDimitry Andric     }
1771349cc55cSDimitry Andric 
17725ffd83dbSDimitry Andric     // Load FP registers from stack.
17735ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::VLLDM))
17745ffd83dbSDimitry Andric         .addReg(ARM::SP)
1775*0fca6ea1SDimitry Andric         .add(predOps(ARMCC::AL))
1776*0fca6ea1SDimitry Andric         .addImm(0); // Represents a pseoudo register list, has no effect on the
1777*0fca6ea1SDimitry Andric                     // encoding.
17785ffd83dbSDimitry Andric 
17795ffd83dbSDimitry Andric     // Pop the stack space
17805ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, DL, TII->get(ARM::tADDspi), ARM::SP)
17815ffd83dbSDimitry Andric         .addReg(ARM::SP)
17825ffd83dbSDimitry Andric         .addImm(CMSE_FP_SAVE_SIZE >> 2)
17835ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
17845ffd83dbSDimitry Andric   } else {
17855ffd83dbSDimitry Andric     // Restore the floating point context.
17865ffd83dbSDimitry Andric     BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(ARM::VLDR_FPCXTS_post),
17875ffd83dbSDimitry Andric             ARM::SP)
17885ffd83dbSDimitry Andric         .addReg(ARM::SP)
17895ffd83dbSDimitry Andric         .addImm(8)
17905ffd83dbSDimitry Andric         .add(predOps(ARMCC::AL));
17915ffd83dbSDimitry Andric 
17925ffd83dbSDimitry Andric     // Pop all the callee-saved registers (s16-s31).
17935ffd83dbSDimitry Andric     MachineInstrBuilder VPOP =
17945ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::VLDMSIA_UPD), ARM::SP)
17955ffd83dbSDimitry Andric             .addReg(ARM::SP)
17965ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
17975ffd83dbSDimitry Andric     for (int Reg = ARM::S16; Reg <= ARM::S31; ++Reg)
17985ffd83dbSDimitry Andric       VPOP.addReg(Reg, RegState::Define);
17995ffd83dbSDimitry Andric   }
18005ffd83dbSDimitry Andric }
18015ffd83dbSDimitry Andric 
18020b57cec5SDimitry Andric /// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
18030b57cec5SDimitry Andric /// possible. This only gets used at -O0 so we don't care about efficiency of
18040b57cec5SDimitry Andric /// the generated code.
18050b57cec5SDimitry Andric bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
18060b57cec5SDimitry Andric                                      MachineBasicBlock::iterator MBBI,
18070b57cec5SDimitry Andric                                      unsigned LdrexOp, unsigned StrexOp,
18080b57cec5SDimitry Andric                                      unsigned UxtOp,
18090b57cec5SDimitry Andric                                      MachineBasicBlock::iterator &NextMBBI) {
18100b57cec5SDimitry Andric   bool IsThumb = STI->isThumb();
1811fcaf7f86SDimitry Andric   bool IsThumb1Only = STI->isThumb1Only();
18120b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
18130b57cec5SDimitry Andric   DebugLoc DL = MI.getDebugLoc();
18140b57cec5SDimitry Andric   const MachineOperand &Dest = MI.getOperand(0);
18158bcb0991SDimitry Andric   Register TempReg = MI.getOperand(1).getReg();
18160b57cec5SDimitry Andric   // Duplicating undef operands into 2 instructions does not guarantee the same
18170b57cec5SDimitry Andric   // value on both; However undef should be replaced by xzr anyway.
18180b57cec5SDimitry Andric   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
18198bcb0991SDimitry Andric   Register AddrReg = MI.getOperand(2).getReg();
18208bcb0991SDimitry Andric   Register DesiredReg = MI.getOperand(3).getReg();
18218bcb0991SDimitry Andric   Register NewReg = MI.getOperand(4).getReg();
18220b57cec5SDimitry Andric 
1823fe6060f1SDimitry Andric   if (IsThumb) {
1824fe6060f1SDimitry Andric     assert(STI->hasV8MBaselineOps() &&
1825fe6060f1SDimitry Andric            "CMP_SWAP not expected to be custom expanded for Thumb1");
1826fe6060f1SDimitry Andric     assert((UxtOp == 0 || UxtOp == ARM::tUXTB || UxtOp == ARM::tUXTH) &&
1827fe6060f1SDimitry Andric            "ARMv8-M.baseline does not have t2UXTB/t2UXTH");
18286e75b2fbSDimitry Andric     assert((UxtOp == 0 || ARM::tGPRRegClass.contains(DesiredReg)) &&
1829fe6060f1SDimitry Andric            "DesiredReg used for UXT op must be tGPR");
1830fe6060f1SDimitry Andric   }
1831fe6060f1SDimitry Andric 
18320b57cec5SDimitry Andric   MachineFunction *MF = MBB.getParent();
18330b57cec5SDimitry Andric   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
18340b57cec5SDimitry Andric   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
18350b57cec5SDimitry Andric   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
18360b57cec5SDimitry Andric 
18370b57cec5SDimitry Andric   MF->insert(++MBB.getIterator(), LoadCmpBB);
18380b57cec5SDimitry Andric   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
18390b57cec5SDimitry Andric   MF->insert(++StoreBB->getIterator(), DoneBB);
18400b57cec5SDimitry Andric 
18410b57cec5SDimitry Andric   if (UxtOp) {
18420b57cec5SDimitry Andric     MachineInstrBuilder MIB =
18430b57cec5SDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
18440b57cec5SDimitry Andric             .addReg(DesiredReg, RegState::Kill);
18450b57cec5SDimitry Andric     if (!IsThumb)
18460b57cec5SDimitry Andric       MIB.addImm(0);
18470b57cec5SDimitry Andric     MIB.add(predOps(ARMCC::AL));
18480b57cec5SDimitry Andric   }
18490b57cec5SDimitry Andric 
18500b57cec5SDimitry Andric   // .Lloadcmp:
18510b57cec5SDimitry Andric   //     ldrex rDest, [rAddr]
18520b57cec5SDimitry Andric   //     cmp rDest, rDesired
18530b57cec5SDimitry Andric   //     bne .Ldone
18540b57cec5SDimitry Andric 
18550b57cec5SDimitry Andric   MachineInstrBuilder MIB;
18560b57cec5SDimitry Andric   MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());
18570b57cec5SDimitry Andric   MIB.addReg(AddrReg);
18580b57cec5SDimitry Andric   if (LdrexOp == ARM::t2LDREX)
18590b57cec5SDimitry Andric     MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.
18600b57cec5SDimitry Andric   MIB.add(predOps(ARMCC::AL));
18610b57cec5SDimitry Andric 
18620b57cec5SDimitry Andric   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
18630b57cec5SDimitry Andric   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
18640b57cec5SDimitry Andric       .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
18650b57cec5SDimitry Andric       .addReg(DesiredReg)
18660b57cec5SDimitry Andric       .add(predOps(ARMCC::AL));
18670b57cec5SDimitry Andric   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
18680b57cec5SDimitry Andric   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
18690b57cec5SDimitry Andric       .addMBB(DoneBB)
18700b57cec5SDimitry Andric       .addImm(ARMCC::NE)
18710b57cec5SDimitry Andric       .addReg(ARM::CPSR, RegState::Kill);
18720b57cec5SDimitry Andric   LoadCmpBB->addSuccessor(DoneBB);
18730b57cec5SDimitry Andric   LoadCmpBB->addSuccessor(StoreBB);
18740b57cec5SDimitry Andric 
18750b57cec5SDimitry Andric   // .Lstore:
18760b57cec5SDimitry Andric   //     strex rTempReg, rNew, [rAddr]
18770b57cec5SDimitry Andric   //     cmp rTempReg, #0
18780b57cec5SDimitry Andric   //     bne .Lloadcmp
18790b57cec5SDimitry Andric   MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)
18800b57cec5SDimitry Andric     .addReg(NewReg)
18810b57cec5SDimitry Andric     .addReg(AddrReg);
18820b57cec5SDimitry Andric   if (StrexOp == ARM::t2STREX)
18830b57cec5SDimitry Andric     MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.
18840b57cec5SDimitry Andric   MIB.add(predOps(ARMCC::AL));
18850b57cec5SDimitry Andric 
1886fcaf7f86SDimitry Andric   unsigned CMPri =
1887fcaf7f86SDimitry Andric       IsThumb ? (IsThumb1Only ? ARM::tCMPi8 : ARM::t2CMPri) : ARM::CMPri;
18880b57cec5SDimitry Andric   BuildMI(StoreBB, DL, TII->get(CMPri))
18890b57cec5SDimitry Andric       .addReg(TempReg, RegState::Kill)
18900b57cec5SDimitry Andric       .addImm(0)
18910b57cec5SDimitry Andric       .add(predOps(ARMCC::AL));
18920b57cec5SDimitry Andric   BuildMI(StoreBB, DL, TII->get(Bcc))
18930b57cec5SDimitry Andric       .addMBB(LoadCmpBB)
18940b57cec5SDimitry Andric       .addImm(ARMCC::NE)
18950b57cec5SDimitry Andric       .addReg(ARM::CPSR, RegState::Kill);
18960b57cec5SDimitry Andric   StoreBB->addSuccessor(LoadCmpBB);
18970b57cec5SDimitry Andric   StoreBB->addSuccessor(DoneBB);
18980b57cec5SDimitry Andric 
18990b57cec5SDimitry Andric   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
19000b57cec5SDimitry Andric   DoneBB->transferSuccessors(&MBB);
19010b57cec5SDimitry Andric 
19020b57cec5SDimitry Andric   MBB.addSuccessor(LoadCmpBB);
19030b57cec5SDimitry Andric 
19040b57cec5SDimitry Andric   NextMBBI = MBB.end();
19050b57cec5SDimitry Andric   MI.eraseFromParent();
19060b57cec5SDimitry Andric 
19070b57cec5SDimitry Andric   // Recompute livein lists.
19080b57cec5SDimitry Andric   LivePhysRegs LiveRegs;
19090b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *DoneBB);
19100b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *StoreBB);
19110b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
19120b57cec5SDimitry Andric   // Do an extra pass around the loop to get loop carried registers right.
19130b57cec5SDimitry Andric   StoreBB->clearLiveIns();
19140b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *StoreBB);
19150b57cec5SDimitry Andric   LoadCmpBB->clearLiveIns();
19160b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
19170b57cec5SDimitry Andric 
19180b57cec5SDimitry Andric   return true;
19190b57cec5SDimitry Andric }
19200b57cec5SDimitry Andric 
19210b57cec5SDimitry Andric /// ARM's ldrexd/strexd take a consecutive register pair (represented as a
19220b57cec5SDimitry Andric /// single GPRPair register), Thumb's take two separate registers so we need to
19230b57cec5SDimitry Andric /// extract the subregs from the pair.
19240b57cec5SDimitry Andric static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
19250b57cec5SDimitry Andric                                 unsigned Flags, bool IsThumb,
19260b57cec5SDimitry Andric                                 const TargetRegisterInfo *TRI) {
19270b57cec5SDimitry Andric   if (IsThumb) {
19288bcb0991SDimitry Andric     Register RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);
19298bcb0991SDimitry Andric     Register RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);
19300b57cec5SDimitry Andric     MIB.addReg(RegLo, Flags);
19310b57cec5SDimitry Andric     MIB.addReg(RegHi, Flags);
19320b57cec5SDimitry Andric   } else
19330b57cec5SDimitry Andric     MIB.addReg(Reg.getReg(), Flags);
19340b57cec5SDimitry Andric }
19350b57cec5SDimitry Andric 
19360b57cec5SDimitry Andric /// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
19370b57cec5SDimitry Andric bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
19380b57cec5SDimitry Andric                                         MachineBasicBlock::iterator MBBI,
19390b57cec5SDimitry Andric                                         MachineBasicBlock::iterator &NextMBBI) {
19400b57cec5SDimitry Andric   bool IsThumb = STI->isThumb();
1941fcaf7f86SDimitry Andric   assert(!STI->isThumb1Only() && "CMP_SWAP_64 unsupported under Thumb1!");
19420b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
19430b57cec5SDimitry Andric   DebugLoc DL = MI.getDebugLoc();
19440b57cec5SDimitry Andric   MachineOperand &Dest = MI.getOperand(0);
19458bcb0991SDimitry Andric   Register TempReg = MI.getOperand(1).getReg();
19460b57cec5SDimitry Andric   // Duplicating undef operands into 2 instructions does not guarantee the same
19470b57cec5SDimitry Andric   // value on both; However undef should be replaced by xzr anyway.
19480b57cec5SDimitry Andric   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
19498bcb0991SDimitry Andric   Register AddrReg = MI.getOperand(2).getReg();
19508bcb0991SDimitry Andric   Register DesiredReg = MI.getOperand(3).getReg();
19510b57cec5SDimitry Andric   MachineOperand New = MI.getOperand(4);
19520b57cec5SDimitry Andric   New.setIsKill(false);
19530b57cec5SDimitry Andric 
19548bcb0991SDimitry Andric   Register DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);
19558bcb0991SDimitry Andric   Register DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);
19568bcb0991SDimitry Andric   Register DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);
19578bcb0991SDimitry Andric   Register DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);
19580b57cec5SDimitry Andric 
19590b57cec5SDimitry Andric   MachineFunction *MF = MBB.getParent();
19600b57cec5SDimitry Andric   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
19610b57cec5SDimitry Andric   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
19620b57cec5SDimitry Andric   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
19630b57cec5SDimitry Andric 
19640b57cec5SDimitry Andric   MF->insert(++MBB.getIterator(), LoadCmpBB);
19650b57cec5SDimitry Andric   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
19660b57cec5SDimitry Andric   MF->insert(++StoreBB->getIterator(), DoneBB);
19670b57cec5SDimitry Andric 
19680b57cec5SDimitry Andric   // .Lloadcmp:
19690b57cec5SDimitry Andric   //     ldrexd rDestLo, rDestHi, [rAddr]
19700b57cec5SDimitry Andric   //     cmp rDestLo, rDesiredLo
19710b57cec5SDimitry Andric   //     sbcs dead rTempReg, rDestHi, rDesiredHi
19720b57cec5SDimitry Andric   //     bne .Ldone
19730b57cec5SDimitry Andric   unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;
19740b57cec5SDimitry Andric   MachineInstrBuilder MIB;
19750b57cec5SDimitry Andric   MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));
19760b57cec5SDimitry Andric   addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);
19770b57cec5SDimitry Andric   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
19780b57cec5SDimitry Andric 
19790b57cec5SDimitry Andric   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
19800b57cec5SDimitry Andric   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
19810b57cec5SDimitry Andric       .addReg(DestLo, getKillRegState(Dest.isDead()))
19820b57cec5SDimitry Andric       .addReg(DesiredLo)
19830b57cec5SDimitry Andric       .add(predOps(ARMCC::AL));
19840b57cec5SDimitry Andric 
19850b57cec5SDimitry Andric   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
19860b57cec5SDimitry Andric       .addReg(DestHi, getKillRegState(Dest.isDead()))
19870b57cec5SDimitry Andric       .addReg(DesiredHi)
19880b57cec5SDimitry Andric       .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);
19890b57cec5SDimitry Andric 
19900b57cec5SDimitry Andric   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
19910b57cec5SDimitry Andric   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
19920b57cec5SDimitry Andric       .addMBB(DoneBB)
19930b57cec5SDimitry Andric       .addImm(ARMCC::NE)
19940b57cec5SDimitry Andric       .addReg(ARM::CPSR, RegState::Kill);
19950b57cec5SDimitry Andric   LoadCmpBB->addSuccessor(DoneBB);
19960b57cec5SDimitry Andric   LoadCmpBB->addSuccessor(StoreBB);
19970b57cec5SDimitry Andric 
19980b57cec5SDimitry Andric   // .Lstore:
19990b57cec5SDimitry Andric   //     strexd rTempReg, rNewLo, rNewHi, [rAddr]
20000b57cec5SDimitry Andric   //     cmp rTempReg, #0
20010b57cec5SDimitry Andric   //     bne .Lloadcmp
20020b57cec5SDimitry Andric   unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;
20030b57cec5SDimitry Andric   MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);
20040b57cec5SDimitry Andric   unsigned Flags = getKillRegState(New.isDead());
20050b57cec5SDimitry Andric   addExclusiveRegPair(MIB, New, Flags, IsThumb, TRI);
20060b57cec5SDimitry Andric   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
20070b57cec5SDimitry Andric 
20080b57cec5SDimitry Andric   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
20090b57cec5SDimitry Andric   BuildMI(StoreBB, DL, TII->get(CMPri))
20100b57cec5SDimitry Andric       .addReg(TempReg, RegState::Kill)
20110b57cec5SDimitry Andric       .addImm(0)
20120b57cec5SDimitry Andric       .add(predOps(ARMCC::AL));
20130b57cec5SDimitry Andric   BuildMI(StoreBB, DL, TII->get(Bcc))
20140b57cec5SDimitry Andric       .addMBB(LoadCmpBB)
20150b57cec5SDimitry Andric       .addImm(ARMCC::NE)
20160b57cec5SDimitry Andric       .addReg(ARM::CPSR, RegState::Kill);
20170b57cec5SDimitry Andric   StoreBB->addSuccessor(LoadCmpBB);
20180b57cec5SDimitry Andric   StoreBB->addSuccessor(DoneBB);
20190b57cec5SDimitry Andric 
20200b57cec5SDimitry Andric   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
20210b57cec5SDimitry Andric   DoneBB->transferSuccessors(&MBB);
20220b57cec5SDimitry Andric 
20230b57cec5SDimitry Andric   MBB.addSuccessor(LoadCmpBB);
20240b57cec5SDimitry Andric 
20250b57cec5SDimitry Andric   NextMBBI = MBB.end();
20260b57cec5SDimitry Andric   MI.eraseFromParent();
20270b57cec5SDimitry Andric 
20280b57cec5SDimitry Andric   // Recompute livein lists.
20290b57cec5SDimitry Andric   LivePhysRegs LiveRegs;
20300b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *DoneBB);
20310b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *StoreBB);
20320b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
20330b57cec5SDimitry Andric   // Do an extra pass around the loop to get loop carried registers right.
20340b57cec5SDimitry Andric   StoreBB->clearLiveIns();
20350b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *StoreBB);
20360b57cec5SDimitry Andric   LoadCmpBB->clearLiveIns();
20370b57cec5SDimitry Andric   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
20380b57cec5SDimitry Andric 
20390b57cec5SDimitry Andric   return true;
20400b57cec5SDimitry Andric }
20410b57cec5SDimitry Andric 
20425ffd83dbSDimitry Andric static void CMSEPushCalleeSaves(const TargetInstrInfo &TII,
20435ffd83dbSDimitry Andric                                 MachineBasicBlock &MBB,
20445ffd83dbSDimitry Andric                                 MachineBasicBlock::iterator MBBI, int JumpReg,
20455ffd83dbSDimitry Andric                                 const LivePhysRegs &LiveRegs, bool Thumb1Only) {
20465ffd83dbSDimitry Andric   const DebugLoc &DL = MBBI->getDebugLoc();
20475ffd83dbSDimitry Andric   if (Thumb1Only) { // push Lo and Hi regs separately
20485ffd83dbSDimitry Andric     MachineInstrBuilder PushMIB =
20495ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH)).add(predOps(ARMCC::AL));
20505ffd83dbSDimitry Andric     for (int Reg = ARM::R4; Reg < ARM::R8; ++Reg) {
20515ffd83dbSDimitry Andric       PushMIB.addReg(
20525ffd83dbSDimitry Andric           Reg, Reg == JumpReg || LiveRegs.contains(Reg) ? 0 : RegState::Undef);
20535ffd83dbSDimitry Andric     }
20545ffd83dbSDimitry Andric 
20555ffd83dbSDimitry Andric     // Thumb1 can only tPUSH low regs, so we copy the high regs to the low
20565ffd83dbSDimitry Andric     // regs that we just saved and push the low regs again, taking care to
20575ffd83dbSDimitry Andric     // not clobber JumpReg. If JumpReg is one of the low registers, push first
20585ffd83dbSDimitry Andric     // the values of r9-r11, and then r8. That would leave them ordered in
20595ffd83dbSDimitry Andric     // memory, and allow us to later pop them with a single instructions.
20605ffd83dbSDimitry Andric     // FIXME: Could also use any of r0-r3 that are free (including in the
20615ffd83dbSDimitry Andric     // first PUSH above).
20625ffd83dbSDimitry Andric     for (int LoReg = ARM::R7, HiReg = ARM::R11; LoReg >= ARM::R4; --LoReg) {
20635ffd83dbSDimitry Andric       if (JumpReg == LoReg)
20645ffd83dbSDimitry Andric         continue;
20655ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), LoReg)
20665ffd83dbSDimitry Andric           .addReg(HiReg, LiveRegs.contains(HiReg) ? 0 : RegState::Undef)
20675ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
20685ffd83dbSDimitry Andric       --HiReg;
20695ffd83dbSDimitry Andric     }
20705ffd83dbSDimitry Andric     MachineInstrBuilder PushMIB2 =
20715ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH)).add(predOps(ARMCC::AL));
20725ffd83dbSDimitry Andric     for (int Reg = ARM::R4; Reg < ARM::R8; ++Reg) {
20735ffd83dbSDimitry Andric       if (Reg == JumpReg)
20745ffd83dbSDimitry Andric         continue;
20755ffd83dbSDimitry Andric       PushMIB2.addReg(Reg, RegState::Kill);
20765ffd83dbSDimitry Andric     }
20775ffd83dbSDimitry Andric 
20785ffd83dbSDimitry Andric     // If we couldn't use a low register for temporary storage (because it was
20795ffd83dbSDimitry Andric     // the JumpReg), use r4 or r5, whichever is not JumpReg. It has already been
20805ffd83dbSDimitry Andric     // saved.
20815ffd83dbSDimitry Andric     if (JumpReg >= ARM::R4 && JumpReg <= ARM::R7) {
20825ffd83dbSDimitry Andric       int LoReg = JumpReg == ARM::R4 ? ARM::R5 : ARM::R4;
20835ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), LoReg)
20845ffd83dbSDimitry Andric           .addReg(ARM::R8, LiveRegs.contains(ARM::R8) ? 0 : RegState::Undef)
20855ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
20865ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII.get(ARM::tPUSH))
20875ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL))
20885ffd83dbSDimitry Andric           .addReg(LoReg, RegState::Kill);
20895ffd83dbSDimitry Andric     }
20905ffd83dbSDimitry Andric   } else { // push Lo and Hi registers with a single instruction
20915ffd83dbSDimitry Andric     MachineInstrBuilder PushMIB =
20925ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII.get(ARM::t2STMDB_UPD), ARM::SP)
20935ffd83dbSDimitry Andric             .addReg(ARM::SP)
20945ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
20955ffd83dbSDimitry Andric     for (int Reg = ARM::R4; Reg < ARM::R12; ++Reg) {
20965ffd83dbSDimitry Andric       PushMIB.addReg(
20975ffd83dbSDimitry Andric           Reg, Reg == JumpReg || LiveRegs.contains(Reg) ? 0 : RegState::Undef);
20985ffd83dbSDimitry Andric     }
20995ffd83dbSDimitry Andric   }
21005ffd83dbSDimitry Andric }
21015ffd83dbSDimitry Andric 
21025ffd83dbSDimitry Andric static void CMSEPopCalleeSaves(const TargetInstrInfo &TII,
21035ffd83dbSDimitry Andric                                MachineBasicBlock &MBB,
21045ffd83dbSDimitry Andric                                MachineBasicBlock::iterator MBBI, int JumpReg,
21055ffd83dbSDimitry Andric                                bool Thumb1Only) {
21065ffd83dbSDimitry Andric   const DebugLoc &DL = MBBI->getDebugLoc();
21075ffd83dbSDimitry Andric   if (Thumb1Only) {
21085ffd83dbSDimitry Andric     MachineInstrBuilder PopMIB =
21095ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));
21105ffd83dbSDimitry Andric     for (int R = 0; R < 4; ++R) {
21115ffd83dbSDimitry Andric       PopMIB.addReg(ARM::R4 + R, RegState::Define);
21125ffd83dbSDimitry Andric       BuildMI(MBB, MBBI, DL, TII.get(ARM::tMOVr), ARM::R8 + R)
21135ffd83dbSDimitry Andric           .addReg(ARM::R4 + R, RegState::Kill)
21145ffd83dbSDimitry Andric           .add(predOps(ARMCC::AL));
21155ffd83dbSDimitry Andric     }
21165ffd83dbSDimitry Andric     MachineInstrBuilder PopMIB2 =
21175ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));
21185ffd83dbSDimitry Andric     for (int R = 0; R < 4; ++R)
21195ffd83dbSDimitry Andric       PopMIB2.addReg(ARM::R4 + R, RegState::Define);
21205ffd83dbSDimitry Andric   } else { // pop Lo and Hi registers with a single instruction
21215ffd83dbSDimitry Andric     MachineInstrBuilder PopMIB =
21225ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII.get(ARM::t2LDMIA_UPD), ARM::SP)
21235ffd83dbSDimitry Andric             .addReg(ARM::SP)
21245ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
21255ffd83dbSDimitry Andric     for (int Reg = ARM::R4; Reg < ARM::R12; ++Reg)
21265ffd83dbSDimitry Andric       PopMIB.addReg(Reg, RegState::Define);
21275ffd83dbSDimitry Andric   }
21285ffd83dbSDimitry Andric }
21290b57cec5SDimitry Andric 
21300b57cec5SDimitry Andric bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
21310b57cec5SDimitry Andric                                MachineBasicBlock::iterator MBBI,
21320b57cec5SDimitry Andric                                MachineBasicBlock::iterator &NextMBBI) {
21330b57cec5SDimitry Andric   MachineInstr &MI = *MBBI;
21340b57cec5SDimitry Andric   unsigned Opcode = MI.getOpcode();
21350b57cec5SDimitry Andric   switch (Opcode) {
21360b57cec5SDimitry Andric     default:
21370b57cec5SDimitry Andric       return false;
21380b57cec5SDimitry Andric 
2139e8d8bef9SDimitry Andric     case ARM::VBSPd:
2140e8d8bef9SDimitry Andric     case ARM::VBSPq: {
2141e8d8bef9SDimitry Andric       Register DstReg = MI.getOperand(0).getReg();
2142e8d8bef9SDimitry Andric       if (DstReg == MI.getOperand(3).getReg()) {
2143e8d8bef9SDimitry Andric         // Expand to VBIT
2144e8d8bef9SDimitry Andric         unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBITd : ARM::VBITq;
2145e8d8bef9SDimitry Andric         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))
2146e8d8bef9SDimitry Andric             .add(MI.getOperand(0))
2147e8d8bef9SDimitry Andric             .add(MI.getOperand(3))
2148e8d8bef9SDimitry Andric             .add(MI.getOperand(2))
2149e8d8bef9SDimitry Andric             .add(MI.getOperand(1))
2150e8d8bef9SDimitry Andric             .addImm(MI.getOperand(4).getImm())
2151e8d8bef9SDimitry Andric             .add(MI.getOperand(5));
2152e8d8bef9SDimitry Andric       } else if (DstReg == MI.getOperand(2).getReg()) {
2153e8d8bef9SDimitry Andric         // Expand to VBIF
2154e8d8bef9SDimitry Andric         unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBIFd : ARM::VBIFq;
2155e8d8bef9SDimitry Andric         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))
2156e8d8bef9SDimitry Andric             .add(MI.getOperand(0))
2157e8d8bef9SDimitry Andric             .add(MI.getOperand(2))
2158e8d8bef9SDimitry Andric             .add(MI.getOperand(3))
2159e8d8bef9SDimitry Andric             .add(MI.getOperand(1))
2160e8d8bef9SDimitry Andric             .addImm(MI.getOperand(4).getImm())
2161e8d8bef9SDimitry Andric             .add(MI.getOperand(5));
2162e8d8bef9SDimitry Andric       } else {
2163e8d8bef9SDimitry Andric         // Expand to VBSL
2164e8d8bef9SDimitry Andric         unsigned NewOpc = Opcode == ARM::VBSPd ? ARM::VBSLd : ARM::VBSLq;
2165e8d8bef9SDimitry Andric         if (DstReg == MI.getOperand(1).getReg()) {
2166e8d8bef9SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))
2167e8d8bef9SDimitry Andric               .add(MI.getOperand(0))
2168e8d8bef9SDimitry Andric               .add(MI.getOperand(1))
2169e8d8bef9SDimitry Andric               .add(MI.getOperand(2))
2170e8d8bef9SDimitry Andric               .add(MI.getOperand(3))
2171e8d8bef9SDimitry Andric               .addImm(MI.getOperand(4).getImm())
2172e8d8bef9SDimitry Andric               .add(MI.getOperand(5));
2173e8d8bef9SDimitry Andric         } else {
2174e8d8bef9SDimitry Andric           // Use move to satisfy constraints
2175e8d8bef9SDimitry Andric           unsigned MoveOpc = Opcode == ARM::VBSPd ? ARM::VORRd : ARM::VORRq;
2176e8d8bef9SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MoveOpc))
2177e8d8bef9SDimitry Andric               .addReg(DstReg,
2178e8d8bef9SDimitry Andric                       RegState::Define |
2179e8d8bef9SDimitry Andric                           getRenamableRegState(MI.getOperand(0).isRenamable()))
2180e8d8bef9SDimitry Andric               .add(MI.getOperand(1))
2181e8d8bef9SDimitry Andric               .add(MI.getOperand(1))
2182e8d8bef9SDimitry Andric               .addImm(MI.getOperand(4).getImm())
2183e8d8bef9SDimitry Andric               .add(MI.getOperand(5));
2184e8d8bef9SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc))
2185e8d8bef9SDimitry Andric               .add(MI.getOperand(0))
2186e8d8bef9SDimitry Andric               .addReg(DstReg,
2187e8d8bef9SDimitry Andric                       RegState::Kill |
2188e8d8bef9SDimitry Andric                           getRenamableRegState(MI.getOperand(0).isRenamable()))
2189e8d8bef9SDimitry Andric               .add(MI.getOperand(2))
2190e8d8bef9SDimitry Andric               .add(MI.getOperand(3))
2191e8d8bef9SDimitry Andric               .addImm(MI.getOperand(4).getImm())
2192e8d8bef9SDimitry Andric               .add(MI.getOperand(5));
2193e8d8bef9SDimitry Andric         }
2194e8d8bef9SDimitry Andric       }
2195e8d8bef9SDimitry Andric       MI.eraseFromParent();
2196e8d8bef9SDimitry Andric       return true;
2197e8d8bef9SDimitry Andric     }
2198e8d8bef9SDimitry Andric 
21990b57cec5SDimitry Andric     case ARM::TCRETURNdi:
2200*0fca6ea1SDimitry Andric     case ARM::TCRETURNri:
2201*0fca6ea1SDimitry Andric     case ARM::TCRETURNrinotr12: {
22020b57cec5SDimitry Andric       MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
220381ad6265SDimitry Andric       if (MBBI->getOpcode() == ARM::SEH_EpilogEnd)
220481ad6265SDimitry Andric         MBBI--;
220581ad6265SDimitry Andric       if (MBBI->getOpcode() == ARM::SEH_Nop_Ret)
220681ad6265SDimitry Andric         MBBI--;
22070b57cec5SDimitry Andric       assert(MBBI->isReturn() &&
22080b57cec5SDimitry Andric              "Can only insert epilog into returning blocks");
22090b57cec5SDimitry Andric       unsigned RetOpcode = MBBI->getOpcode();
22100b57cec5SDimitry Andric       DebugLoc dl = MBBI->getDebugLoc();
22110b57cec5SDimitry Andric       const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
22120b57cec5SDimitry Andric           MBB.getParent()->getSubtarget().getInstrInfo());
22130b57cec5SDimitry Andric 
22140b57cec5SDimitry Andric       // Tail call return: adjust the stack pointer and jump to callee.
22150b57cec5SDimitry Andric       MBBI = MBB.getLastNonDebugInstr();
221681ad6265SDimitry Andric       if (MBBI->getOpcode() == ARM::SEH_EpilogEnd)
221781ad6265SDimitry Andric         MBBI--;
221881ad6265SDimitry Andric       if (MBBI->getOpcode() == ARM::SEH_Nop_Ret)
221981ad6265SDimitry Andric         MBBI--;
22200b57cec5SDimitry Andric       MachineOperand &JumpTarget = MBBI->getOperand(0);
22210b57cec5SDimitry Andric 
22220b57cec5SDimitry Andric       // Jump to label or value in register.
22230b57cec5SDimitry Andric       if (RetOpcode == ARM::TCRETURNdi) {
222481ad6265SDimitry Andric         MachineFunction *MF = MBB.getParent();
222581ad6265SDimitry Andric         bool NeedsWinCFI = MF->getTarget().getMCAsmInfo()->usesWindowsCFI() &&
222681ad6265SDimitry Andric                            MF->getFunction().needsUnwindTableEntry();
22270b57cec5SDimitry Andric         unsigned TCOpcode =
22280b57cec5SDimitry Andric             STI->isThumb()
222981ad6265SDimitry Andric                 ? ((STI->isTargetMachO() || NeedsWinCFI) ? ARM::tTAILJMPd
223081ad6265SDimitry Andric                                                          : ARM::tTAILJMPdND)
22310b57cec5SDimitry Andric                 : ARM::TAILJMPd;
22320b57cec5SDimitry Andric         MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
22330b57cec5SDimitry Andric         if (JumpTarget.isGlobal())
22340b57cec5SDimitry Andric           MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
22350b57cec5SDimitry Andric                                JumpTarget.getTargetFlags());
22360b57cec5SDimitry Andric         else {
22370b57cec5SDimitry Andric           assert(JumpTarget.isSymbol());
22380b57cec5SDimitry Andric           MIB.addExternalSymbol(JumpTarget.getSymbolName(),
22390b57cec5SDimitry Andric                                 JumpTarget.getTargetFlags());
22400b57cec5SDimitry Andric         }
22410b57cec5SDimitry Andric 
22420b57cec5SDimitry Andric         // Add the default predicate in Thumb mode.
22430b57cec5SDimitry Andric         if (STI->isThumb())
22440b57cec5SDimitry Andric           MIB.add(predOps(ARMCC::AL));
2245*0fca6ea1SDimitry Andric       } else if (RetOpcode == ARM::TCRETURNri ||
2246*0fca6ea1SDimitry Andric                  RetOpcode == ARM::TCRETURNrinotr12) {
22470b57cec5SDimitry Andric         unsigned Opcode =
22480b57cec5SDimitry Andric           STI->isThumb() ? ARM::tTAILJMPr
22490b57cec5SDimitry Andric                          : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
22500b57cec5SDimitry Andric         BuildMI(MBB, MBBI, dl,
22510b57cec5SDimitry Andric                 TII.get(Opcode))
22520b57cec5SDimitry Andric             .addReg(JumpTarget.getReg(), RegState::Kill);
22530b57cec5SDimitry Andric       }
22540b57cec5SDimitry Andric 
22550b57cec5SDimitry Andric       auto NewMI = std::prev(MBBI);
2256fe6060f1SDimitry Andric       for (unsigned i = 2, e = MBBI->getNumOperands(); i != e; ++i)
22570b57cec5SDimitry Andric         NewMI->addOperand(MBBI->getOperand(i));
22580b57cec5SDimitry Andric 
22598bcb0991SDimitry Andric 
22608bcb0991SDimitry Andric       // Update call site info and delete the pseudo instruction TCRETURN.
22615ffd83dbSDimitry Andric       if (MI.isCandidateForCallSiteEntry())
22625ffd83dbSDimitry Andric         MI.getMF()->moveCallSiteInfo(&MI, &*NewMI);
226306c3fb27SDimitry Andric       // Copy nomerge flag over to new instruction.
226406c3fb27SDimitry Andric       if (MI.getFlag(MachineInstr::NoMerge))
226506c3fb27SDimitry Andric         NewMI->setFlag(MachineInstr::NoMerge);
22660b57cec5SDimitry Andric       MBB.erase(MBBI);
22678bcb0991SDimitry Andric 
22680b57cec5SDimitry Andric       MBBI = NewMI;
22690b57cec5SDimitry Andric       return true;
22700b57cec5SDimitry Andric     }
22715ffd83dbSDimitry Andric     case ARM::tBXNS_RET: {
22720eae32dcSDimitry Andric       // For v8.0-M.Main we need to authenticate LR before clearing FPRs, which
22730eae32dcSDimitry Andric       // uses R12 as a scratch register.
22740eae32dcSDimitry Andric       if (!STI->hasV8_1MMainlineOps() && AFI->shouldSignReturnAddress())
22750eae32dcSDimitry Andric         BuildMI(MBB, MBBI, DebugLoc(), TII->get(ARM::t2AUT));
22760eae32dcSDimitry Andric 
22775ffd83dbSDimitry Andric       MachineBasicBlock &AfterBB = CMSEClearFPRegs(MBB, MBBI);
22785ffd83dbSDimitry Andric 
22795ffd83dbSDimitry Andric       if (STI->hasV8_1MMainlineOps()) {
22805ffd83dbSDimitry Andric         // Restore the non-secure floating point context.
22815ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
22825ffd83dbSDimitry Andric                 TII->get(ARM::VLDR_FPCXTNS_post), ARM::SP)
22835ffd83dbSDimitry Andric             .addReg(ARM::SP)
22845ffd83dbSDimitry Andric             .addImm(4)
22855ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
22860eae32dcSDimitry Andric 
22870eae32dcSDimitry Andric         if (AFI->shouldSignReturnAddress())
22880eae32dcSDimitry Andric           BuildMI(AfterBB, AfterBB.end(), DebugLoc(), TII->get(ARM::t2AUT));
22895ffd83dbSDimitry Andric       }
22905ffd83dbSDimitry Andric 
22915ffd83dbSDimitry Andric       // Clear all GPR that are not a use of the return instruction.
22925ffd83dbSDimitry Andric       assert(llvm::all_of(MBBI->operands(), [](const MachineOperand &Op) {
22935ffd83dbSDimitry Andric         return !Op.isReg() || Op.getReg() != ARM::R12;
22945ffd83dbSDimitry Andric       }));
22955ffd83dbSDimitry Andric       SmallVector<unsigned, 5> ClearRegs;
22965ffd83dbSDimitry Andric       determineGPRegsToClear(
22975ffd83dbSDimitry Andric           *MBBI, {ARM::R0, ARM::R1, ARM::R2, ARM::R3, ARM::R12}, ClearRegs);
22985ffd83dbSDimitry Andric       CMSEClearGPRegs(AfterBB, AfterBB.end(), MBBI->getDebugLoc(), ClearRegs,
22995ffd83dbSDimitry Andric                       ARM::LR);
23005ffd83dbSDimitry Andric 
23015ffd83dbSDimitry Andric       MachineInstrBuilder NewMI =
23025ffd83dbSDimitry Andric           BuildMI(AfterBB, AfterBB.end(), MBBI->getDebugLoc(),
23035ffd83dbSDimitry Andric                   TII->get(ARM::tBXNS))
23045ffd83dbSDimitry Andric               .addReg(ARM::LR)
23055ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL));
23065ffd83dbSDimitry Andric       for (const MachineOperand &Op : MI.operands())
23075ffd83dbSDimitry Andric         NewMI->addOperand(Op);
23085ffd83dbSDimitry Andric       MI.eraseFromParent();
23095ffd83dbSDimitry Andric       return true;
23105ffd83dbSDimitry Andric     }
23115ffd83dbSDimitry Andric     case ARM::tBLXNS_CALL: {
23125ffd83dbSDimitry Andric       DebugLoc DL = MBBI->getDebugLoc();
231304eeddc0SDimitry Andric       Register JumpReg = MBBI->getOperand(0).getReg();
23145ffd83dbSDimitry Andric 
23155ffd83dbSDimitry Andric       // Figure out which registers are live at the point immediately before the
23165ffd83dbSDimitry Andric       // call. When we indiscriminately push a set of registers, the live
23175ffd83dbSDimitry Andric       // registers are added as ordinary use operands, whereas dead registers
23185ffd83dbSDimitry Andric       // are "undef".
23195ffd83dbSDimitry Andric       LivePhysRegs LiveRegs(*TRI);
23205ffd83dbSDimitry Andric       LiveRegs.addLiveOuts(MBB);
23215ffd83dbSDimitry Andric       for (const MachineInstr &MI : make_range(MBB.rbegin(), MBBI.getReverse()))
23225ffd83dbSDimitry Andric         LiveRegs.stepBackward(MI);
23235ffd83dbSDimitry Andric       LiveRegs.stepBackward(*MBBI);
23245ffd83dbSDimitry Andric 
23255ffd83dbSDimitry Andric       CMSEPushCalleeSaves(*TII, MBB, MBBI, JumpReg, LiveRegs,
23265ffd83dbSDimitry Andric                           AFI->isThumb1OnlyFunction());
23275ffd83dbSDimitry Andric 
23285ffd83dbSDimitry Andric       SmallVector<unsigned, 16> ClearRegs;
23295ffd83dbSDimitry Andric       determineGPRegsToClear(*MBBI,
23305ffd83dbSDimitry Andric                              {ARM::R0, ARM::R1, ARM::R2, ARM::R3, ARM::R4,
23315ffd83dbSDimitry Andric                               ARM::R5, ARM::R6, ARM::R7, ARM::R8, ARM::R9,
23325ffd83dbSDimitry Andric                               ARM::R10, ARM::R11, ARM::R12},
23335ffd83dbSDimitry Andric                              ClearRegs);
23345ffd83dbSDimitry Andric       auto OriginalClearRegs = ClearRegs;
23355ffd83dbSDimitry Andric 
23365ffd83dbSDimitry Andric       // Get the first cleared register as a scratch (to use later with tBIC).
23375ffd83dbSDimitry Andric       // We need to use the first so we can ensure it is a low register.
23385ffd83dbSDimitry Andric       unsigned ScratchReg = ClearRegs.front();
23395ffd83dbSDimitry Andric 
23405ffd83dbSDimitry Andric       // Clear LSB of JumpReg
23415ffd83dbSDimitry Andric       if (AFI->isThumb2Function()) {
23425ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::t2BICri), JumpReg)
23435ffd83dbSDimitry Andric             .addReg(JumpReg)
23445ffd83dbSDimitry Andric             .addImm(1)
23455ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL))
23465ffd83dbSDimitry Andric             .add(condCodeOp());
23475ffd83dbSDimitry Andric       } else {
23485ffd83dbSDimitry Andric         // We need to use an extra register to cope with 8M Baseline,
23495ffd83dbSDimitry Andric         // since we have saved all of the registers we are ok to trash a non
23505ffd83dbSDimitry Andric         // argument register here.
23515ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::tMOVi8), ScratchReg)
23525ffd83dbSDimitry Andric             .add(condCodeOp())
23535ffd83dbSDimitry Andric             .addImm(1)
23545ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
23555ffd83dbSDimitry Andric         BuildMI(MBB, MBBI, DL, TII->get(ARM::tBIC), JumpReg)
23565ffd83dbSDimitry Andric             .addReg(ARM::CPSR, RegState::Define)
23575ffd83dbSDimitry Andric             .addReg(JumpReg)
23585ffd83dbSDimitry Andric             .addReg(ScratchReg)
23595ffd83dbSDimitry Andric             .add(predOps(ARMCC::AL));
23605ffd83dbSDimitry Andric       }
23615ffd83dbSDimitry Andric 
23625ffd83dbSDimitry Andric       CMSESaveClearFPRegs(MBB, MBBI, DL, LiveRegs,
23635ffd83dbSDimitry Andric                           ClearRegs); // save+clear FP regs with ClearRegs
23645ffd83dbSDimitry Andric       CMSEClearGPRegs(MBB, MBBI, DL, ClearRegs, JumpReg);
23655ffd83dbSDimitry Andric 
23665ffd83dbSDimitry Andric       const MachineInstrBuilder NewCall =
23675ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, DL, TII->get(ARM::tBLXNSr))
23685ffd83dbSDimitry Andric               .add(predOps(ARMCC::AL))
23695ffd83dbSDimitry Andric               .addReg(JumpReg, RegState::Kill);
23705ffd83dbSDimitry Andric 
23714824e7fdSDimitry Andric       for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
23724824e7fdSDimitry Andric         NewCall->addOperand(MO);
23735ffd83dbSDimitry Andric       if (MI.isCandidateForCallSiteEntry())
23745ffd83dbSDimitry Andric         MI.getMF()->moveCallSiteInfo(&MI, NewCall.getInstr());
23755ffd83dbSDimitry Andric 
23765ffd83dbSDimitry Andric       CMSERestoreFPRegs(MBB, MBBI, DL, OriginalClearRegs); // restore FP registers
23775ffd83dbSDimitry Andric 
23785ffd83dbSDimitry Andric       CMSEPopCalleeSaves(*TII, MBB, MBBI, JumpReg, AFI->isThumb1OnlyFunction());
23795ffd83dbSDimitry Andric 
23805ffd83dbSDimitry Andric       MI.eraseFromParent();
23815ffd83dbSDimitry Andric       return true;
23825ffd83dbSDimitry Andric     }
2383480093f4SDimitry Andric     case ARM::VMOVHcc:
23840b57cec5SDimitry Andric     case ARM::VMOVScc:
23850b57cec5SDimitry Andric     case ARM::VMOVDcc: {
2386480093f4SDimitry Andric       unsigned newOpc = Opcode != ARM::VMOVDcc ? ARM::VMOVS : ARM::VMOVD;
23870b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),
23880b57cec5SDimitry Andric               MI.getOperand(1).getReg())
23890b57cec5SDimitry Andric           .add(MI.getOperand(2))
23900b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm()) // 'pred'
23910b57cec5SDimitry Andric           .add(MI.getOperand(4))
23920b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
23930b57cec5SDimitry Andric 
23940b57cec5SDimitry Andric       MI.eraseFromParent();
23950b57cec5SDimitry Andric       return true;
23960b57cec5SDimitry Andric     }
23970b57cec5SDimitry Andric     case ARM::t2MOVCCr:
23980b57cec5SDimitry Andric     case ARM::MOVCCr: {
23990b57cec5SDimitry Andric       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;
24000b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
24010b57cec5SDimitry Andric               MI.getOperand(1).getReg())
24020b57cec5SDimitry Andric           .add(MI.getOperand(2))
24030b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm()) // 'pred'
24040b57cec5SDimitry Andric           .add(MI.getOperand(4))
24050b57cec5SDimitry Andric           .add(condCodeOp()) // 's' bit
24060b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24070b57cec5SDimitry Andric 
24080b57cec5SDimitry Andric       MI.eraseFromParent();
24090b57cec5SDimitry Andric       return true;
24100b57cec5SDimitry Andric     }
24110b57cec5SDimitry Andric     case ARM::MOVCCsi: {
24120b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
24130b57cec5SDimitry Andric               (MI.getOperand(1).getReg()))
24140b57cec5SDimitry Andric           .add(MI.getOperand(2))
24150b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm())
24160b57cec5SDimitry Andric           .addImm(MI.getOperand(4).getImm()) // 'pred'
24170b57cec5SDimitry Andric           .add(MI.getOperand(5))
24180b57cec5SDimitry Andric           .add(condCodeOp()) // 's' bit
24190b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24200b57cec5SDimitry Andric 
24210b57cec5SDimitry Andric       MI.eraseFromParent();
24220b57cec5SDimitry Andric       return true;
24230b57cec5SDimitry Andric     }
24240b57cec5SDimitry Andric     case ARM::MOVCCsr: {
24250b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
24260b57cec5SDimitry Andric               (MI.getOperand(1).getReg()))
24270b57cec5SDimitry Andric           .add(MI.getOperand(2))
24280b57cec5SDimitry Andric           .add(MI.getOperand(3))
24290b57cec5SDimitry Andric           .addImm(MI.getOperand(4).getImm())
24300b57cec5SDimitry Andric           .addImm(MI.getOperand(5).getImm()) // 'pred'
24310b57cec5SDimitry Andric           .add(MI.getOperand(6))
24320b57cec5SDimitry Andric           .add(condCodeOp()) // 's' bit
24330b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24340b57cec5SDimitry Andric 
24350b57cec5SDimitry Andric       MI.eraseFromParent();
24360b57cec5SDimitry Andric       return true;
24370b57cec5SDimitry Andric     }
24380b57cec5SDimitry Andric     case ARM::t2MOVCCi16:
24390b57cec5SDimitry Andric     case ARM::MOVCCi16: {
24400b57cec5SDimitry Andric       unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;
24410b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
24420b57cec5SDimitry Andric               MI.getOperand(1).getReg())
24430b57cec5SDimitry Andric           .addImm(MI.getOperand(2).getImm())
24440b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm()) // 'pred'
24450b57cec5SDimitry Andric           .add(MI.getOperand(4))
24460b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24470b57cec5SDimitry Andric       MI.eraseFromParent();
24480b57cec5SDimitry Andric       return true;
24490b57cec5SDimitry Andric     }
24500b57cec5SDimitry Andric     case ARM::t2MOVCCi:
24510b57cec5SDimitry Andric     case ARM::MOVCCi: {
24520b57cec5SDimitry Andric       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;
24530b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
24540b57cec5SDimitry Andric               MI.getOperand(1).getReg())
24550b57cec5SDimitry Andric           .addImm(MI.getOperand(2).getImm())
24560b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm()) // 'pred'
24570b57cec5SDimitry Andric           .add(MI.getOperand(4))
24580b57cec5SDimitry Andric           .add(condCodeOp()) // 's' bit
24590b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24600b57cec5SDimitry Andric 
24610b57cec5SDimitry Andric       MI.eraseFromParent();
24620b57cec5SDimitry Andric       return true;
24630b57cec5SDimitry Andric     }
24640b57cec5SDimitry Andric     case ARM::t2MVNCCi:
24650b57cec5SDimitry Andric     case ARM::MVNCCi: {
24660b57cec5SDimitry Andric       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;
24670b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
24680b57cec5SDimitry Andric               MI.getOperand(1).getReg())
24690b57cec5SDimitry Andric           .addImm(MI.getOperand(2).getImm())
24700b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm()) // 'pred'
24710b57cec5SDimitry Andric           .add(MI.getOperand(4))
24720b57cec5SDimitry Andric           .add(condCodeOp()) // 's' bit
24730b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24740b57cec5SDimitry Andric 
24750b57cec5SDimitry Andric       MI.eraseFromParent();
24760b57cec5SDimitry Andric       return true;
24770b57cec5SDimitry Andric     }
24780b57cec5SDimitry Andric     case ARM::t2MOVCClsl:
24790b57cec5SDimitry Andric     case ARM::t2MOVCClsr:
24800b57cec5SDimitry Andric     case ARM::t2MOVCCasr:
24810b57cec5SDimitry Andric     case ARM::t2MOVCCror: {
24820b57cec5SDimitry Andric       unsigned NewOpc;
24830b57cec5SDimitry Andric       switch (Opcode) {
24840b57cec5SDimitry Andric       case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;
24850b57cec5SDimitry Andric       case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;
24860b57cec5SDimitry Andric       case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;
24870b57cec5SDimitry Andric       case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;
24880b57cec5SDimitry Andric       default: llvm_unreachable("unexpeced conditional move");
24890b57cec5SDimitry Andric       }
24900b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
24910b57cec5SDimitry Andric               MI.getOperand(1).getReg())
24920b57cec5SDimitry Andric           .add(MI.getOperand(2))
24930b57cec5SDimitry Andric           .addImm(MI.getOperand(3).getImm())
24940b57cec5SDimitry Andric           .addImm(MI.getOperand(4).getImm()) // 'pred'
24950b57cec5SDimitry Andric           .add(MI.getOperand(5))
24960b57cec5SDimitry Andric           .add(condCodeOp()) // 's' bit
24970b57cec5SDimitry Andric           .add(makeImplicit(MI.getOperand(1)));
24980b57cec5SDimitry Andric       MI.eraseFromParent();
24990b57cec5SDimitry Andric       return true;
25000b57cec5SDimitry Andric     }
25010b57cec5SDimitry Andric     case ARM::Int_eh_sjlj_dispatchsetup: {
25020b57cec5SDimitry Andric       MachineFunction &MF = *MI.getParent()->getParent();
25030b57cec5SDimitry Andric       const ARMBaseInstrInfo *AII =
25040b57cec5SDimitry Andric         static_cast<const ARMBaseInstrInfo*>(TII);
25050b57cec5SDimitry Andric       const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
25060b57cec5SDimitry Andric       // For functions using a base pointer, we rematerialize it (via the frame
25070b57cec5SDimitry Andric       // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
25080b57cec5SDimitry Andric       // for us. Otherwise, expand to nothing.
25090b57cec5SDimitry Andric       if (RI.hasBasePointer(MF)) {
25100b57cec5SDimitry Andric         int32_t NumBytes = AFI->getFramePtrSpillOffset();
25118bcb0991SDimitry Andric         Register FramePtr = RI.getFrameRegister(MF);
25120b57cec5SDimitry Andric         assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
25130b57cec5SDimitry Andric                "base pointer without frame pointer?");
25140b57cec5SDimitry Andric 
25150b57cec5SDimitry Andric         if (AFI->isThumb2Function()) {
25160b57cec5SDimitry Andric           emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
25170b57cec5SDimitry Andric                                  FramePtr, -NumBytes, ARMCC::AL, 0, *TII);
25180b57cec5SDimitry Andric         } else if (AFI->isThumbFunction()) {
25190b57cec5SDimitry Andric           emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
25200b57cec5SDimitry Andric                                     FramePtr, -NumBytes, *TII, RI);
25210b57cec5SDimitry Andric         } else {
25220b57cec5SDimitry Andric           emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
25230b57cec5SDimitry Andric                                   FramePtr, -NumBytes, ARMCC::AL, 0,
25240b57cec5SDimitry Andric                                   *TII);
25250b57cec5SDimitry Andric         }
25260b57cec5SDimitry Andric         // If there's dynamic realignment, adjust for it.
2527fe6060f1SDimitry Andric         if (RI.hasStackRealignment(MF)) {
25280b57cec5SDimitry Andric           MachineFrameInfo &MFI = MF.getFrameInfo();
25295ffd83dbSDimitry Andric           Align MaxAlign = MFI.getMaxAlign();
25300b57cec5SDimitry Andric           assert (!AFI->isThumb1OnlyFunction());
25310b57cec5SDimitry Andric           // Emit bic r6, r6, MaxAlign
25325ffd83dbSDimitry Andric           assert(MaxAlign <= Align(256) &&
25335ffd83dbSDimitry Andric                  "The BIC instruction cannot encode "
25340b57cec5SDimitry Andric                  "immediates larger than 256 with all lower "
25350b57cec5SDimitry Andric                  "bits set.");
25360b57cec5SDimitry Andric           unsigned bicOpc = AFI->isThumbFunction() ?
25370b57cec5SDimitry Andric             ARM::t2BICri : ARM::BICri;
25380b57cec5SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)
25390b57cec5SDimitry Andric               .addReg(ARM::R6, RegState::Kill)
25405ffd83dbSDimitry Andric               .addImm(MaxAlign.value() - 1)
25410b57cec5SDimitry Andric               .add(predOps(ARMCC::AL))
25420b57cec5SDimitry Andric               .add(condCodeOp());
25430b57cec5SDimitry Andric         }
25440b57cec5SDimitry Andric       }
25450b57cec5SDimitry Andric       MI.eraseFromParent();
25460b57cec5SDimitry Andric       return true;
25470b57cec5SDimitry Andric     }
25480b57cec5SDimitry Andric 
254906c3fb27SDimitry Andric     case ARM::MOVsrl_glue:
255006c3fb27SDimitry Andric     case ARM::MOVsra_glue: {
25510b57cec5SDimitry Andric       // These are just fancy MOVs instructions.
25520b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
25530b57cec5SDimitry Andric               MI.getOperand(0).getReg())
25540b57cec5SDimitry Andric           .add(MI.getOperand(1))
25550b57cec5SDimitry Andric           .addImm(ARM_AM::getSORegOpc(
255606c3fb27SDimitry Andric               (Opcode == ARM::MOVsrl_glue ? ARM_AM::lsr : ARM_AM::asr), 1))
25570b57cec5SDimitry Andric           .add(predOps(ARMCC::AL))
25580b57cec5SDimitry Andric           .addReg(ARM::CPSR, RegState::Define);
25590b57cec5SDimitry Andric       MI.eraseFromParent();
25600b57cec5SDimitry Andric       return true;
25610b57cec5SDimitry Andric     }
25620b57cec5SDimitry Andric     case ARM::RRX: {
25630b57cec5SDimitry Andric       // This encodes as "MOVs Rd, Rm, rrx
25640b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
25650b57cec5SDimitry Andric               MI.getOperand(0).getReg())
25660b57cec5SDimitry Andric           .add(MI.getOperand(1))
25670b57cec5SDimitry Andric           .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))
25680b57cec5SDimitry Andric           .add(predOps(ARMCC::AL))
256906c3fb27SDimitry Andric           .add(condCodeOp())
257006c3fb27SDimitry Andric           .copyImplicitOps(MI);
25710b57cec5SDimitry Andric       MI.eraseFromParent();
25720b57cec5SDimitry Andric       return true;
25730b57cec5SDimitry Andric     }
25740b57cec5SDimitry Andric     case ARM::tTPsoft:
25750b57cec5SDimitry Andric     case ARM::TPsoft: {
25760b57cec5SDimitry Andric       const bool Thumb = Opcode == ARM::tTPsoft;
25770b57cec5SDimitry Andric 
25780b57cec5SDimitry Andric       MachineInstrBuilder MIB;
25790b57cec5SDimitry Andric       MachineFunction *MF = MBB.getParent();
25805ffd83dbSDimitry Andric       if (STI->genLongCalls()) {
25810b57cec5SDimitry Andric         MachineConstantPool *MCP = MF->getConstantPool();
25820b57cec5SDimitry Andric         unsigned PCLabelID = AFI->createPICLabelUId();
25830b57cec5SDimitry Andric         MachineConstantPoolValue *CPV =
25840b57cec5SDimitry Andric             ARMConstantPoolSymbol::Create(MF->getFunction().getContext(),
25850b57cec5SDimitry Andric                                           "__aeabi_read_tp", PCLabelID, 0);
25868bcb0991SDimitry Andric         Register Reg = MI.getOperand(0).getReg();
25875ffd83dbSDimitry Andric         MIB =
25885ffd83dbSDimitry Andric             BuildMI(MBB, MBBI, MI.getDebugLoc(),
25890b57cec5SDimitry Andric                     TII->get(Thumb ? ARM::tLDRpci : ARM::LDRi12), Reg)
25905ffd83dbSDimitry Andric                 .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, Align(4)));
25910b57cec5SDimitry Andric         if (!Thumb)
25920b57cec5SDimitry Andric           MIB.addImm(0);
25930b57cec5SDimitry Andric         MIB.add(predOps(ARMCC::AL));
25940b57cec5SDimitry Andric 
2595e8d8bef9SDimitry Andric         MIB =
2596e8d8bef9SDimitry Andric             BuildMI(MBB, MBBI, MI.getDebugLoc(),
2597e8d8bef9SDimitry Andric                     TII->get(Thumb ? gettBLXrOpcode(*MF) : getBLXOpcode(*MF)));
25980b57cec5SDimitry Andric         if (Thumb)
25990b57cec5SDimitry Andric           MIB.add(predOps(ARMCC::AL));
26000b57cec5SDimitry Andric         MIB.addReg(Reg, RegState::Kill);
26010b57cec5SDimitry Andric       } else {
26020b57cec5SDimitry Andric         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
26030b57cec5SDimitry Andric                       TII->get(Thumb ? ARM::tBL : ARM::BL));
26040b57cec5SDimitry Andric         if (Thumb)
26050b57cec5SDimitry Andric           MIB.add(predOps(ARMCC::AL));
26060b57cec5SDimitry Andric         MIB.addExternalSymbol("__aeabi_read_tp", 0);
26070b57cec5SDimitry Andric       }
26080b57cec5SDimitry Andric 
26090b57cec5SDimitry Andric       MIB.cloneMemRefs(MI);
261006c3fb27SDimitry Andric       MIB.copyImplicitOps(MI);
26115ffd83dbSDimitry Andric       // Update the call site info.
26125ffd83dbSDimitry Andric       if (MI.isCandidateForCallSiteEntry())
26135ffd83dbSDimitry Andric         MF->moveCallSiteInfo(&MI, &*MIB);
26140b57cec5SDimitry Andric       MI.eraseFromParent();
26150b57cec5SDimitry Andric       return true;
26160b57cec5SDimitry Andric     }
26170b57cec5SDimitry Andric     case ARM::tLDRpci_pic:
26180b57cec5SDimitry Andric     case ARM::t2LDRpci_pic: {
26190b57cec5SDimitry Andric       unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
26200b57cec5SDimitry Andric         ? ARM::tLDRpci : ARM::t2LDRpci;
26218bcb0991SDimitry Andric       Register DstReg = MI.getOperand(0).getReg();
26220b57cec5SDimitry Andric       bool DstIsDead = MI.getOperand(0).isDead();
26230b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)
26240b57cec5SDimitry Andric           .add(MI.getOperand(1))
262506c3fb27SDimitry Andric           .add(predOps(ARMCC::AL))
262606c3fb27SDimitry Andric           .cloneMemRefs(MI)
262706c3fb27SDimitry Andric           .copyImplicitOps(MI);
26280b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))
26290b57cec5SDimitry Andric           .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
26300b57cec5SDimitry Andric           .addReg(DstReg)
263106c3fb27SDimitry Andric           .add(MI.getOperand(2))
263206c3fb27SDimitry Andric           .copyImplicitOps(MI);
26330b57cec5SDimitry Andric       MI.eraseFromParent();
26340b57cec5SDimitry Andric       return true;
26350b57cec5SDimitry Andric     }
26360b57cec5SDimitry Andric 
26370b57cec5SDimitry Andric     case ARM::LDRLIT_ga_abs:
26380b57cec5SDimitry Andric     case ARM::LDRLIT_ga_pcrel:
26390b57cec5SDimitry Andric     case ARM::LDRLIT_ga_pcrel_ldr:
26400b57cec5SDimitry Andric     case ARM::tLDRLIT_ga_abs:
26414824e7fdSDimitry Andric     case ARM::t2LDRLIT_ga_pcrel:
26420b57cec5SDimitry Andric     case ARM::tLDRLIT_ga_pcrel: {
26438bcb0991SDimitry Andric       Register DstReg = MI.getOperand(0).getReg();
26440b57cec5SDimitry Andric       bool DstIsDead = MI.getOperand(0).isDead();
26450b57cec5SDimitry Andric       const MachineOperand &MO1 = MI.getOperand(1);
26460b57cec5SDimitry Andric       auto Flags = MO1.getTargetFlags();
26470b57cec5SDimitry Andric       const GlobalValue *GV = MO1.getGlobal();
26484824e7fdSDimitry Andric       bool IsARM = Opcode != ARM::tLDRLIT_ga_pcrel &&
26494824e7fdSDimitry Andric                    Opcode != ARM::tLDRLIT_ga_abs &&
26504824e7fdSDimitry Andric                    Opcode != ARM::t2LDRLIT_ga_pcrel;
26510b57cec5SDimitry Andric       bool IsPIC =
26520b57cec5SDimitry Andric           Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;
26530b57cec5SDimitry Andric       unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;
26544824e7fdSDimitry Andric       if (Opcode == ARM::t2LDRLIT_ga_pcrel)
26554824e7fdSDimitry Andric         LDRLITOpc = ARM::t2LDRpci;
26560b57cec5SDimitry Andric       unsigned PICAddOpc =
26570b57cec5SDimitry Andric           IsARM
26580b57cec5SDimitry Andric               ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
26590b57cec5SDimitry Andric               : ARM::tPICADD;
26600b57cec5SDimitry Andric 
26610b57cec5SDimitry Andric       // We need a new const-pool entry to load from.
26620b57cec5SDimitry Andric       MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
26630b57cec5SDimitry Andric       unsigned ARMPCLabelIndex = 0;
26640b57cec5SDimitry Andric       MachineConstantPoolValue *CPV;
26650b57cec5SDimitry Andric 
26660b57cec5SDimitry Andric       if (IsPIC) {
26670b57cec5SDimitry Andric         unsigned PCAdj = IsARM ? 8 : 4;
26680b57cec5SDimitry Andric         auto Modifier = (Flags & ARMII::MO_GOT)
26690b57cec5SDimitry Andric                             ? ARMCP::GOT_PREL
26700b57cec5SDimitry Andric                             : ARMCP::no_modifier;
26710b57cec5SDimitry Andric         ARMPCLabelIndex = AFI->createPICLabelUId();
26720b57cec5SDimitry Andric         CPV = ARMConstantPoolConstant::Create(
26730b57cec5SDimitry Andric             GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,
26740b57cec5SDimitry Andric             /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);
26750b57cec5SDimitry Andric       } else
26760b57cec5SDimitry Andric         CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);
26770b57cec5SDimitry Andric 
26780b57cec5SDimitry Andric       MachineInstrBuilder MIB =
26790b57cec5SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)
26805ffd83dbSDimitry Andric               .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, Align(4)));
26810b57cec5SDimitry Andric       if (IsARM)
26820b57cec5SDimitry Andric         MIB.addImm(0);
26830b57cec5SDimitry Andric       MIB.add(predOps(ARMCC::AL));
26840b57cec5SDimitry Andric 
26850b57cec5SDimitry Andric       if (IsPIC) {
26860b57cec5SDimitry Andric         MachineInstrBuilder MIB =
26870b57cec5SDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))
26880b57cec5SDimitry Andric             .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
26890b57cec5SDimitry Andric             .addReg(DstReg)
26900b57cec5SDimitry Andric             .addImm(ARMPCLabelIndex);
26910b57cec5SDimitry Andric 
26920b57cec5SDimitry Andric         if (IsARM)
26930b57cec5SDimitry Andric           MIB.add(predOps(ARMCC::AL));
26940b57cec5SDimitry Andric       }
26950b57cec5SDimitry Andric 
26960b57cec5SDimitry Andric       MI.eraseFromParent();
26970b57cec5SDimitry Andric       return true;
26980b57cec5SDimitry Andric     }
26990b57cec5SDimitry Andric     case ARM::MOV_ga_pcrel:
27000b57cec5SDimitry Andric     case ARM::MOV_ga_pcrel_ldr:
27010b57cec5SDimitry Andric     case ARM::t2MOV_ga_pcrel: {
27020b57cec5SDimitry Andric       // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
27030b57cec5SDimitry Andric       unsigned LabelId = AFI->createPICLabelUId();
27048bcb0991SDimitry Andric       Register DstReg = MI.getOperand(0).getReg();
27050b57cec5SDimitry Andric       bool DstIsDead = MI.getOperand(0).isDead();
27060b57cec5SDimitry Andric       const MachineOperand &MO1 = MI.getOperand(1);
27070b57cec5SDimitry Andric       const GlobalValue *GV = MO1.getGlobal();
27080b57cec5SDimitry Andric       unsigned TF = MO1.getTargetFlags();
27090b57cec5SDimitry Andric       bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
27100b57cec5SDimitry Andric       unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;
27110b57cec5SDimitry Andric       unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;
27120b57cec5SDimitry Andric       unsigned LO16TF = TF | ARMII::MO_LO16;
27130b57cec5SDimitry Andric       unsigned HI16TF = TF | ARMII::MO_HI16;
27140b57cec5SDimitry Andric       unsigned PICAddOpc = isARM
27150b57cec5SDimitry Andric         ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
27160b57cec5SDimitry Andric         : ARM::tPICADD;
271706c3fb27SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg)
27180b57cec5SDimitry Andric           .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)
271906c3fb27SDimitry Andric           .addImm(LabelId)
272006c3fb27SDimitry Andric           .copyImplicitOps(MI);
27210b57cec5SDimitry Andric 
27220b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)
27230b57cec5SDimitry Andric           .addReg(DstReg)
27240b57cec5SDimitry Andric           .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)
272506c3fb27SDimitry Andric           .addImm(LabelId)
272606c3fb27SDimitry Andric           .copyImplicitOps(MI);
27270b57cec5SDimitry Andric 
27280b57cec5SDimitry Andric       MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
27290b57cec5SDimitry Andric                                          TII->get(PICAddOpc))
27300b57cec5SDimitry Andric         .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
27310b57cec5SDimitry Andric         .addReg(DstReg).addImm(LabelId);
27320b57cec5SDimitry Andric       if (isARM) {
27330b57cec5SDimitry Andric         MIB3.add(predOps(ARMCC::AL));
27340b57cec5SDimitry Andric         if (Opcode == ARM::MOV_ga_pcrel_ldr)
27350b57cec5SDimitry Andric           MIB3.cloneMemRefs(MI);
27360b57cec5SDimitry Andric       }
273706c3fb27SDimitry Andric       MIB3.copyImplicitOps(MI);
27380b57cec5SDimitry Andric       MI.eraseFromParent();
27390b57cec5SDimitry Andric       return true;
27400b57cec5SDimitry Andric     }
27410b57cec5SDimitry Andric 
27420b57cec5SDimitry Andric     case ARM::MOVi32imm:
27430b57cec5SDimitry Andric     case ARM::MOVCCi32imm:
27440b57cec5SDimitry Andric     case ARM::t2MOVi32imm:
27450b57cec5SDimitry Andric     case ARM::t2MOVCCi32imm:
27460b57cec5SDimitry Andric       ExpandMOV32BitImm(MBB, MBBI);
27470b57cec5SDimitry Andric       return true;
27480b57cec5SDimitry Andric 
274906c3fb27SDimitry Andric     case ARM::tMOVi32imm:
275006c3fb27SDimitry Andric       ExpandTMOV32BitImm(MBB, MBBI);
275106c3fb27SDimitry Andric       return true;
275206c3fb27SDimitry Andric 
275306c3fb27SDimitry Andric     case ARM::tLEApcrelJT:
275406c3fb27SDimitry Andric       // Inline jump tables are handled in ARMAsmPrinter.
275506c3fb27SDimitry Andric       if (MI.getMF()->getJumpTableInfo()->getEntryKind() ==
275606c3fb27SDimitry Andric           MachineJumpTableInfo::EK_Inline)
275706c3fb27SDimitry Andric         return false;
275806c3fb27SDimitry Andric 
275906c3fb27SDimitry Andric       // Use a 32-bit immediate move to generate the address of the jump table.
276006c3fb27SDimitry Andric       assert(STI->isThumb() && "Non-inline jump tables expected only in thumb");
276106c3fb27SDimitry Andric       ExpandTMOV32BitImm(MBB, MBBI);
276206c3fb27SDimitry Andric       return true;
276306c3fb27SDimitry Andric 
27640b57cec5SDimitry Andric     case ARM::SUBS_PC_LR: {
27650b57cec5SDimitry Andric       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)
27660b57cec5SDimitry Andric           .addReg(ARM::LR)
27670b57cec5SDimitry Andric           .add(MI.getOperand(0))
27680b57cec5SDimitry Andric           .add(MI.getOperand(1))
27690b57cec5SDimitry Andric           .add(MI.getOperand(2))
277006c3fb27SDimitry Andric           .addReg(ARM::CPSR, RegState::Undef)
277106c3fb27SDimitry Andric           .copyImplicitOps(MI);
27720b57cec5SDimitry Andric       MI.eraseFromParent();
27730b57cec5SDimitry Andric       return true;
27740b57cec5SDimitry Andric     }
27750b57cec5SDimitry Andric     case ARM::VLDMQIA: {
27760b57cec5SDimitry Andric       unsigned NewOpc = ARM::VLDMDIA;
27770b57cec5SDimitry Andric       MachineInstrBuilder MIB =
27780b57cec5SDimitry Andric         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
27790b57cec5SDimitry Andric       unsigned OpIdx = 0;
27800b57cec5SDimitry Andric 
27810b57cec5SDimitry Andric       // Grab the Q register destination.
27820b57cec5SDimitry Andric       bool DstIsDead = MI.getOperand(OpIdx).isDead();
27838bcb0991SDimitry Andric       Register DstReg = MI.getOperand(OpIdx++).getReg();
27840b57cec5SDimitry Andric 
27850b57cec5SDimitry Andric       // Copy the source register.
27860b57cec5SDimitry Andric       MIB.add(MI.getOperand(OpIdx++));
27870b57cec5SDimitry Andric 
27880b57cec5SDimitry Andric       // Copy the predicate operands.
27890b57cec5SDimitry Andric       MIB.add(MI.getOperand(OpIdx++));
27900b57cec5SDimitry Andric       MIB.add(MI.getOperand(OpIdx++));
27910b57cec5SDimitry Andric 
27920b57cec5SDimitry Andric       // Add the destination operands (D subregs).
27938bcb0991SDimitry Andric       Register D0 = TRI->getSubReg(DstReg, ARM::dsub_0);
27948bcb0991SDimitry Andric       Register D1 = TRI->getSubReg(DstReg, ARM::dsub_1);
27950b57cec5SDimitry Andric       MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))
27960b57cec5SDimitry Andric         .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
27970b57cec5SDimitry Andric 
27980b57cec5SDimitry Andric       // Add an implicit def for the super-register.
27990b57cec5SDimitry Andric       MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
280006c3fb27SDimitry Andric       MIB.copyImplicitOps(MI);
28010b57cec5SDimitry Andric       MIB.cloneMemRefs(MI);
28020b57cec5SDimitry Andric       MI.eraseFromParent();
28030b57cec5SDimitry Andric       return true;
28040b57cec5SDimitry Andric     }
28050b57cec5SDimitry Andric 
28060b57cec5SDimitry Andric     case ARM::VSTMQIA: {
28070b57cec5SDimitry Andric       unsigned NewOpc = ARM::VSTMDIA;
28080b57cec5SDimitry Andric       MachineInstrBuilder MIB =
28090b57cec5SDimitry Andric         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
28100b57cec5SDimitry Andric       unsigned OpIdx = 0;
28110b57cec5SDimitry Andric 
28120b57cec5SDimitry Andric       // Grab the Q register source.
28130b57cec5SDimitry Andric       bool SrcIsKill = MI.getOperand(OpIdx).isKill();
28148bcb0991SDimitry Andric       Register SrcReg = MI.getOperand(OpIdx++).getReg();
28150b57cec5SDimitry Andric 
28160b57cec5SDimitry Andric       // Copy the destination register.
28170b57cec5SDimitry Andric       MachineOperand Dst(MI.getOperand(OpIdx++));
28180b57cec5SDimitry Andric       MIB.add(Dst);
28190b57cec5SDimitry Andric 
28200b57cec5SDimitry Andric       // Copy the predicate operands.
28210b57cec5SDimitry Andric       MIB.add(MI.getOperand(OpIdx++));
28220b57cec5SDimitry Andric       MIB.add(MI.getOperand(OpIdx++));
28230b57cec5SDimitry Andric 
28240b57cec5SDimitry Andric       // Add the source operands (D subregs).
28258bcb0991SDimitry Andric       Register D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);
28268bcb0991SDimitry Andric       Register D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);
28270b57cec5SDimitry Andric       MIB.addReg(D0, SrcIsKill ? RegState::Kill : 0)
28280b57cec5SDimitry Andric          .addReg(D1, SrcIsKill ? RegState::Kill : 0);
28290b57cec5SDimitry Andric 
28300b57cec5SDimitry Andric       if (SrcIsKill)      // Add an implicit kill for the Q register.
28310b57cec5SDimitry Andric         MIB->addRegisterKilled(SrcReg, TRI, true);
28320b57cec5SDimitry Andric 
283306c3fb27SDimitry Andric       MIB.copyImplicitOps(MI);
28340b57cec5SDimitry Andric       MIB.cloneMemRefs(MI);
28350b57cec5SDimitry Andric       MI.eraseFromParent();
28360b57cec5SDimitry Andric       return true;
28370b57cec5SDimitry Andric     }
28380b57cec5SDimitry Andric 
28390b57cec5SDimitry Andric     case ARM::VLD2q8Pseudo:
28400b57cec5SDimitry Andric     case ARM::VLD2q16Pseudo:
28410b57cec5SDimitry Andric     case ARM::VLD2q32Pseudo:
28420b57cec5SDimitry Andric     case ARM::VLD2q8PseudoWB_fixed:
28430b57cec5SDimitry Andric     case ARM::VLD2q16PseudoWB_fixed:
28440b57cec5SDimitry Andric     case ARM::VLD2q32PseudoWB_fixed:
28450b57cec5SDimitry Andric     case ARM::VLD2q8PseudoWB_register:
28460b57cec5SDimitry Andric     case ARM::VLD2q16PseudoWB_register:
28470b57cec5SDimitry Andric     case ARM::VLD2q32PseudoWB_register:
28480b57cec5SDimitry Andric     case ARM::VLD3d8Pseudo:
28490b57cec5SDimitry Andric     case ARM::VLD3d16Pseudo:
28500b57cec5SDimitry Andric     case ARM::VLD3d32Pseudo:
28510b57cec5SDimitry Andric     case ARM::VLD1d8TPseudo:
2852fe6060f1SDimitry Andric     case ARM::VLD1d8TPseudoWB_fixed:
2853fe6060f1SDimitry Andric     case ARM::VLD1d8TPseudoWB_register:
28540b57cec5SDimitry Andric     case ARM::VLD1d16TPseudo:
2855fe6060f1SDimitry Andric     case ARM::VLD1d16TPseudoWB_fixed:
2856fe6060f1SDimitry Andric     case ARM::VLD1d16TPseudoWB_register:
28570b57cec5SDimitry Andric     case ARM::VLD1d32TPseudo:
2858fe6060f1SDimitry Andric     case ARM::VLD1d32TPseudoWB_fixed:
2859fe6060f1SDimitry Andric     case ARM::VLD1d32TPseudoWB_register:
28600b57cec5SDimitry Andric     case ARM::VLD1d64TPseudo:
28610b57cec5SDimitry Andric     case ARM::VLD1d64TPseudoWB_fixed:
28620b57cec5SDimitry Andric     case ARM::VLD1d64TPseudoWB_register:
28630b57cec5SDimitry Andric     case ARM::VLD3d8Pseudo_UPD:
28640b57cec5SDimitry Andric     case ARM::VLD3d16Pseudo_UPD:
28650b57cec5SDimitry Andric     case ARM::VLD3d32Pseudo_UPD:
28660b57cec5SDimitry Andric     case ARM::VLD3q8Pseudo_UPD:
28670b57cec5SDimitry Andric     case ARM::VLD3q16Pseudo_UPD:
28680b57cec5SDimitry Andric     case ARM::VLD3q32Pseudo_UPD:
28690b57cec5SDimitry Andric     case ARM::VLD3q8oddPseudo:
28700b57cec5SDimitry Andric     case ARM::VLD3q16oddPseudo:
28710b57cec5SDimitry Andric     case ARM::VLD3q32oddPseudo:
28720b57cec5SDimitry Andric     case ARM::VLD3q8oddPseudo_UPD:
28730b57cec5SDimitry Andric     case ARM::VLD3q16oddPseudo_UPD:
28740b57cec5SDimitry Andric     case ARM::VLD3q32oddPseudo_UPD:
28750b57cec5SDimitry Andric     case ARM::VLD4d8Pseudo:
28760b57cec5SDimitry Andric     case ARM::VLD4d16Pseudo:
28770b57cec5SDimitry Andric     case ARM::VLD4d32Pseudo:
28780b57cec5SDimitry Andric     case ARM::VLD1d8QPseudo:
2879fe6060f1SDimitry Andric     case ARM::VLD1d8QPseudoWB_fixed:
2880fe6060f1SDimitry Andric     case ARM::VLD1d8QPseudoWB_register:
28810b57cec5SDimitry Andric     case ARM::VLD1d16QPseudo:
2882fe6060f1SDimitry Andric     case ARM::VLD1d16QPseudoWB_fixed:
2883fe6060f1SDimitry Andric     case ARM::VLD1d16QPseudoWB_register:
28840b57cec5SDimitry Andric     case ARM::VLD1d32QPseudo:
2885fe6060f1SDimitry Andric     case ARM::VLD1d32QPseudoWB_fixed:
2886fe6060f1SDimitry Andric     case ARM::VLD1d32QPseudoWB_register:
28870b57cec5SDimitry Andric     case ARM::VLD1d64QPseudo:
28880b57cec5SDimitry Andric     case ARM::VLD1d64QPseudoWB_fixed:
28890b57cec5SDimitry Andric     case ARM::VLD1d64QPseudoWB_register:
28900b57cec5SDimitry Andric     case ARM::VLD1q8HighQPseudo:
2891fe6060f1SDimitry Andric     case ARM::VLD1q8HighQPseudo_UPD:
28920b57cec5SDimitry Andric     case ARM::VLD1q8LowQPseudo_UPD:
28930b57cec5SDimitry Andric     case ARM::VLD1q8HighTPseudo:
2894fe6060f1SDimitry Andric     case ARM::VLD1q8HighTPseudo_UPD:
28950b57cec5SDimitry Andric     case ARM::VLD1q8LowTPseudo_UPD:
28960b57cec5SDimitry Andric     case ARM::VLD1q16HighQPseudo:
2897fe6060f1SDimitry Andric     case ARM::VLD1q16HighQPseudo_UPD:
28980b57cec5SDimitry Andric     case ARM::VLD1q16LowQPseudo_UPD:
28990b57cec5SDimitry Andric     case ARM::VLD1q16HighTPseudo:
2900fe6060f1SDimitry Andric     case ARM::VLD1q16HighTPseudo_UPD:
29010b57cec5SDimitry Andric     case ARM::VLD1q16LowTPseudo_UPD:
29020b57cec5SDimitry Andric     case ARM::VLD1q32HighQPseudo:
2903fe6060f1SDimitry Andric     case ARM::VLD1q32HighQPseudo_UPD:
29040b57cec5SDimitry Andric     case ARM::VLD1q32LowQPseudo_UPD:
29050b57cec5SDimitry Andric     case ARM::VLD1q32HighTPseudo:
2906fe6060f1SDimitry Andric     case ARM::VLD1q32HighTPseudo_UPD:
29070b57cec5SDimitry Andric     case ARM::VLD1q32LowTPseudo_UPD:
29080b57cec5SDimitry Andric     case ARM::VLD1q64HighQPseudo:
2909fe6060f1SDimitry Andric     case ARM::VLD1q64HighQPseudo_UPD:
29100b57cec5SDimitry Andric     case ARM::VLD1q64LowQPseudo_UPD:
29110b57cec5SDimitry Andric     case ARM::VLD1q64HighTPseudo:
2912fe6060f1SDimitry Andric     case ARM::VLD1q64HighTPseudo_UPD:
29130b57cec5SDimitry Andric     case ARM::VLD1q64LowTPseudo_UPD:
29140b57cec5SDimitry Andric     case ARM::VLD4d8Pseudo_UPD:
29150b57cec5SDimitry Andric     case ARM::VLD4d16Pseudo_UPD:
29160b57cec5SDimitry Andric     case ARM::VLD4d32Pseudo_UPD:
29170b57cec5SDimitry Andric     case ARM::VLD4q8Pseudo_UPD:
29180b57cec5SDimitry Andric     case ARM::VLD4q16Pseudo_UPD:
29190b57cec5SDimitry Andric     case ARM::VLD4q32Pseudo_UPD:
29200b57cec5SDimitry Andric     case ARM::VLD4q8oddPseudo:
29210b57cec5SDimitry Andric     case ARM::VLD4q16oddPseudo:
29220b57cec5SDimitry Andric     case ARM::VLD4q32oddPseudo:
29230b57cec5SDimitry Andric     case ARM::VLD4q8oddPseudo_UPD:
29240b57cec5SDimitry Andric     case ARM::VLD4q16oddPseudo_UPD:
29250b57cec5SDimitry Andric     case ARM::VLD4q32oddPseudo_UPD:
29260b57cec5SDimitry Andric     case ARM::VLD3DUPd8Pseudo:
29270b57cec5SDimitry Andric     case ARM::VLD3DUPd16Pseudo:
29280b57cec5SDimitry Andric     case ARM::VLD3DUPd32Pseudo:
29290b57cec5SDimitry Andric     case ARM::VLD3DUPd8Pseudo_UPD:
29300b57cec5SDimitry Andric     case ARM::VLD3DUPd16Pseudo_UPD:
29310b57cec5SDimitry Andric     case ARM::VLD3DUPd32Pseudo_UPD:
29320b57cec5SDimitry Andric     case ARM::VLD4DUPd8Pseudo:
29330b57cec5SDimitry Andric     case ARM::VLD4DUPd16Pseudo:
29340b57cec5SDimitry Andric     case ARM::VLD4DUPd32Pseudo:
29350b57cec5SDimitry Andric     case ARM::VLD4DUPd8Pseudo_UPD:
29360b57cec5SDimitry Andric     case ARM::VLD4DUPd16Pseudo_UPD:
29370b57cec5SDimitry Andric     case ARM::VLD4DUPd32Pseudo_UPD:
29380b57cec5SDimitry Andric     case ARM::VLD2DUPq8EvenPseudo:
29390b57cec5SDimitry Andric     case ARM::VLD2DUPq8OddPseudo:
29400b57cec5SDimitry Andric     case ARM::VLD2DUPq16EvenPseudo:
29410b57cec5SDimitry Andric     case ARM::VLD2DUPq16OddPseudo:
29420b57cec5SDimitry Andric     case ARM::VLD2DUPq32EvenPseudo:
29430b57cec5SDimitry Andric     case ARM::VLD2DUPq32OddPseudo:
2944fe6060f1SDimitry Andric     case ARM::VLD2DUPq8OddPseudoWB_fixed:
2945fe6060f1SDimitry Andric     case ARM::VLD2DUPq8OddPseudoWB_register:
2946fe6060f1SDimitry Andric     case ARM::VLD2DUPq16OddPseudoWB_fixed:
2947fe6060f1SDimitry Andric     case ARM::VLD2DUPq16OddPseudoWB_register:
2948fe6060f1SDimitry Andric     case ARM::VLD2DUPq32OddPseudoWB_fixed:
2949fe6060f1SDimitry Andric     case ARM::VLD2DUPq32OddPseudoWB_register:
29500b57cec5SDimitry Andric     case ARM::VLD3DUPq8EvenPseudo:
29510b57cec5SDimitry Andric     case ARM::VLD3DUPq8OddPseudo:
29520b57cec5SDimitry Andric     case ARM::VLD3DUPq16EvenPseudo:
29530b57cec5SDimitry Andric     case ARM::VLD3DUPq16OddPseudo:
29540b57cec5SDimitry Andric     case ARM::VLD3DUPq32EvenPseudo:
29550b57cec5SDimitry Andric     case ARM::VLD3DUPq32OddPseudo:
2956fe6060f1SDimitry Andric     case ARM::VLD3DUPq8OddPseudo_UPD:
2957fe6060f1SDimitry Andric     case ARM::VLD3DUPq16OddPseudo_UPD:
2958fe6060f1SDimitry Andric     case ARM::VLD3DUPq32OddPseudo_UPD:
29590b57cec5SDimitry Andric     case ARM::VLD4DUPq8EvenPseudo:
29600b57cec5SDimitry Andric     case ARM::VLD4DUPq8OddPseudo:
29610b57cec5SDimitry Andric     case ARM::VLD4DUPq16EvenPseudo:
29620b57cec5SDimitry Andric     case ARM::VLD4DUPq16OddPseudo:
29630b57cec5SDimitry Andric     case ARM::VLD4DUPq32EvenPseudo:
29640b57cec5SDimitry Andric     case ARM::VLD4DUPq32OddPseudo:
2965fe6060f1SDimitry Andric     case ARM::VLD4DUPq8OddPseudo_UPD:
2966fe6060f1SDimitry Andric     case ARM::VLD4DUPq16OddPseudo_UPD:
2967fe6060f1SDimitry Andric     case ARM::VLD4DUPq32OddPseudo_UPD:
29680b57cec5SDimitry Andric       ExpandVLD(MBBI);
29690b57cec5SDimitry Andric       return true;
29700b57cec5SDimitry Andric 
29710b57cec5SDimitry Andric     case ARM::VST2q8Pseudo:
29720b57cec5SDimitry Andric     case ARM::VST2q16Pseudo:
29730b57cec5SDimitry Andric     case ARM::VST2q32Pseudo:
29740b57cec5SDimitry Andric     case ARM::VST2q8PseudoWB_fixed:
29750b57cec5SDimitry Andric     case ARM::VST2q16PseudoWB_fixed:
29760b57cec5SDimitry Andric     case ARM::VST2q32PseudoWB_fixed:
29770b57cec5SDimitry Andric     case ARM::VST2q8PseudoWB_register:
29780b57cec5SDimitry Andric     case ARM::VST2q16PseudoWB_register:
29790b57cec5SDimitry Andric     case ARM::VST2q32PseudoWB_register:
29800b57cec5SDimitry Andric     case ARM::VST3d8Pseudo:
29810b57cec5SDimitry Andric     case ARM::VST3d16Pseudo:
29820b57cec5SDimitry Andric     case ARM::VST3d32Pseudo:
29830b57cec5SDimitry Andric     case ARM::VST1d8TPseudo:
2984fe6060f1SDimitry Andric     case ARM::VST1d8TPseudoWB_fixed:
2985fe6060f1SDimitry Andric     case ARM::VST1d8TPseudoWB_register:
29860b57cec5SDimitry Andric     case ARM::VST1d16TPseudo:
2987fe6060f1SDimitry Andric     case ARM::VST1d16TPseudoWB_fixed:
2988fe6060f1SDimitry Andric     case ARM::VST1d16TPseudoWB_register:
29890b57cec5SDimitry Andric     case ARM::VST1d32TPseudo:
2990fe6060f1SDimitry Andric     case ARM::VST1d32TPseudoWB_fixed:
2991fe6060f1SDimitry Andric     case ARM::VST1d32TPseudoWB_register:
29920b57cec5SDimitry Andric     case ARM::VST1d64TPseudo:
2993fe6060f1SDimitry Andric     case ARM::VST1d64TPseudoWB_fixed:
2994fe6060f1SDimitry Andric     case ARM::VST1d64TPseudoWB_register:
29950b57cec5SDimitry Andric     case ARM::VST3d8Pseudo_UPD:
29960b57cec5SDimitry Andric     case ARM::VST3d16Pseudo_UPD:
29970b57cec5SDimitry Andric     case ARM::VST3d32Pseudo_UPD:
29980b57cec5SDimitry Andric     case ARM::VST3q8Pseudo_UPD:
29990b57cec5SDimitry Andric     case ARM::VST3q16Pseudo_UPD:
30000b57cec5SDimitry Andric     case ARM::VST3q32Pseudo_UPD:
30010b57cec5SDimitry Andric     case ARM::VST3q8oddPseudo:
30020b57cec5SDimitry Andric     case ARM::VST3q16oddPseudo:
30030b57cec5SDimitry Andric     case ARM::VST3q32oddPseudo:
30040b57cec5SDimitry Andric     case ARM::VST3q8oddPseudo_UPD:
30050b57cec5SDimitry Andric     case ARM::VST3q16oddPseudo_UPD:
30060b57cec5SDimitry Andric     case ARM::VST3q32oddPseudo_UPD:
30070b57cec5SDimitry Andric     case ARM::VST4d8Pseudo:
30080b57cec5SDimitry Andric     case ARM::VST4d16Pseudo:
30090b57cec5SDimitry Andric     case ARM::VST4d32Pseudo:
30100b57cec5SDimitry Andric     case ARM::VST1d8QPseudo:
3011fe6060f1SDimitry Andric     case ARM::VST1d8QPseudoWB_fixed:
3012fe6060f1SDimitry Andric     case ARM::VST1d8QPseudoWB_register:
30130b57cec5SDimitry Andric     case ARM::VST1d16QPseudo:
3014fe6060f1SDimitry Andric     case ARM::VST1d16QPseudoWB_fixed:
3015fe6060f1SDimitry Andric     case ARM::VST1d16QPseudoWB_register:
30160b57cec5SDimitry Andric     case ARM::VST1d32QPseudo:
3017fe6060f1SDimitry Andric     case ARM::VST1d32QPseudoWB_fixed:
3018fe6060f1SDimitry Andric     case ARM::VST1d32QPseudoWB_register:
30190b57cec5SDimitry Andric     case ARM::VST1d64QPseudo:
3020fe6060f1SDimitry Andric     case ARM::VST1d64QPseudoWB_fixed:
3021fe6060f1SDimitry Andric     case ARM::VST1d64QPseudoWB_register:
30220b57cec5SDimitry Andric     case ARM::VST4d8Pseudo_UPD:
30230b57cec5SDimitry Andric     case ARM::VST4d16Pseudo_UPD:
30240b57cec5SDimitry Andric     case ARM::VST4d32Pseudo_UPD:
30250b57cec5SDimitry Andric     case ARM::VST1q8HighQPseudo:
30260b57cec5SDimitry Andric     case ARM::VST1q8LowQPseudo_UPD:
30270b57cec5SDimitry Andric     case ARM::VST1q8HighTPseudo:
30280b57cec5SDimitry Andric     case ARM::VST1q8LowTPseudo_UPD:
30290b57cec5SDimitry Andric     case ARM::VST1q16HighQPseudo:
30300b57cec5SDimitry Andric     case ARM::VST1q16LowQPseudo_UPD:
30310b57cec5SDimitry Andric     case ARM::VST1q16HighTPseudo:
30320b57cec5SDimitry Andric     case ARM::VST1q16LowTPseudo_UPD:
30330b57cec5SDimitry Andric     case ARM::VST1q32HighQPseudo:
30340b57cec5SDimitry Andric     case ARM::VST1q32LowQPseudo_UPD:
30350b57cec5SDimitry Andric     case ARM::VST1q32HighTPseudo:
30360b57cec5SDimitry Andric     case ARM::VST1q32LowTPseudo_UPD:
30370b57cec5SDimitry Andric     case ARM::VST1q64HighQPseudo:
30380b57cec5SDimitry Andric     case ARM::VST1q64LowQPseudo_UPD:
30390b57cec5SDimitry Andric     case ARM::VST1q64HighTPseudo:
30400b57cec5SDimitry Andric     case ARM::VST1q64LowTPseudo_UPD:
3041fe6060f1SDimitry Andric     case ARM::VST1q8HighTPseudo_UPD:
3042fe6060f1SDimitry Andric     case ARM::VST1q16HighTPseudo_UPD:
3043fe6060f1SDimitry Andric     case ARM::VST1q32HighTPseudo_UPD:
3044fe6060f1SDimitry Andric     case ARM::VST1q64HighTPseudo_UPD:
3045fe6060f1SDimitry Andric     case ARM::VST1q8HighQPseudo_UPD:
3046fe6060f1SDimitry Andric     case ARM::VST1q16HighQPseudo_UPD:
3047fe6060f1SDimitry Andric     case ARM::VST1q32HighQPseudo_UPD:
3048fe6060f1SDimitry Andric     case ARM::VST1q64HighQPseudo_UPD:
30490b57cec5SDimitry Andric     case ARM::VST4q8Pseudo_UPD:
30500b57cec5SDimitry Andric     case ARM::VST4q16Pseudo_UPD:
30510b57cec5SDimitry Andric     case ARM::VST4q32Pseudo_UPD:
30520b57cec5SDimitry Andric     case ARM::VST4q8oddPseudo:
30530b57cec5SDimitry Andric     case ARM::VST4q16oddPseudo:
30540b57cec5SDimitry Andric     case ARM::VST4q32oddPseudo:
30550b57cec5SDimitry Andric     case ARM::VST4q8oddPseudo_UPD:
30560b57cec5SDimitry Andric     case ARM::VST4q16oddPseudo_UPD:
30570b57cec5SDimitry Andric     case ARM::VST4q32oddPseudo_UPD:
30580b57cec5SDimitry Andric       ExpandVST(MBBI);
30590b57cec5SDimitry Andric       return true;
30600b57cec5SDimitry Andric 
30610b57cec5SDimitry Andric     case ARM::VLD1LNq8Pseudo:
30620b57cec5SDimitry Andric     case ARM::VLD1LNq16Pseudo:
30630b57cec5SDimitry Andric     case ARM::VLD1LNq32Pseudo:
30640b57cec5SDimitry Andric     case ARM::VLD1LNq8Pseudo_UPD:
30650b57cec5SDimitry Andric     case ARM::VLD1LNq16Pseudo_UPD:
30660b57cec5SDimitry Andric     case ARM::VLD1LNq32Pseudo_UPD:
30670b57cec5SDimitry Andric     case ARM::VLD2LNd8Pseudo:
30680b57cec5SDimitry Andric     case ARM::VLD2LNd16Pseudo:
30690b57cec5SDimitry Andric     case ARM::VLD2LNd32Pseudo:
30700b57cec5SDimitry Andric     case ARM::VLD2LNq16Pseudo:
30710b57cec5SDimitry Andric     case ARM::VLD2LNq32Pseudo:
30720b57cec5SDimitry Andric     case ARM::VLD2LNd8Pseudo_UPD:
30730b57cec5SDimitry Andric     case ARM::VLD2LNd16Pseudo_UPD:
30740b57cec5SDimitry Andric     case ARM::VLD2LNd32Pseudo_UPD:
30750b57cec5SDimitry Andric     case ARM::VLD2LNq16Pseudo_UPD:
30760b57cec5SDimitry Andric     case ARM::VLD2LNq32Pseudo_UPD:
30770b57cec5SDimitry Andric     case ARM::VLD3LNd8Pseudo:
30780b57cec5SDimitry Andric     case ARM::VLD3LNd16Pseudo:
30790b57cec5SDimitry Andric     case ARM::VLD3LNd32Pseudo:
30800b57cec5SDimitry Andric     case ARM::VLD3LNq16Pseudo:
30810b57cec5SDimitry Andric     case ARM::VLD3LNq32Pseudo:
30820b57cec5SDimitry Andric     case ARM::VLD3LNd8Pseudo_UPD:
30830b57cec5SDimitry Andric     case ARM::VLD3LNd16Pseudo_UPD:
30840b57cec5SDimitry Andric     case ARM::VLD3LNd32Pseudo_UPD:
30850b57cec5SDimitry Andric     case ARM::VLD3LNq16Pseudo_UPD:
30860b57cec5SDimitry Andric     case ARM::VLD3LNq32Pseudo_UPD:
30870b57cec5SDimitry Andric     case ARM::VLD4LNd8Pseudo:
30880b57cec5SDimitry Andric     case ARM::VLD4LNd16Pseudo:
30890b57cec5SDimitry Andric     case ARM::VLD4LNd32Pseudo:
30900b57cec5SDimitry Andric     case ARM::VLD4LNq16Pseudo:
30910b57cec5SDimitry Andric     case ARM::VLD4LNq32Pseudo:
30920b57cec5SDimitry Andric     case ARM::VLD4LNd8Pseudo_UPD:
30930b57cec5SDimitry Andric     case ARM::VLD4LNd16Pseudo_UPD:
30940b57cec5SDimitry Andric     case ARM::VLD4LNd32Pseudo_UPD:
30950b57cec5SDimitry Andric     case ARM::VLD4LNq16Pseudo_UPD:
30960b57cec5SDimitry Andric     case ARM::VLD4LNq32Pseudo_UPD:
30970b57cec5SDimitry Andric     case ARM::VST1LNq8Pseudo:
30980b57cec5SDimitry Andric     case ARM::VST1LNq16Pseudo:
30990b57cec5SDimitry Andric     case ARM::VST1LNq32Pseudo:
31000b57cec5SDimitry Andric     case ARM::VST1LNq8Pseudo_UPD:
31010b57cec5SDimitry Andric     case ARM::VST1LNq16Pseudo_UPD:
31020b57cec5SDimitry Andric     case ARM::VST1LNq32Pseudo_UPD:
31030b57cec5SDimitry Andric     case ARM::VST2LNd8Pseudo:
31040b57cec5SDimitry Andric     case ARM::VST2LNd16Pseudo:
31050b57cec5SDimitry Andric     case ARM::VST2LNd32Pseudo:
31060b57cec5SDimitry Andric     case ARM::VST2LNq16Pseudo:
31070b57cec5SDimitry Andric     case ARM::VST2LNq32Pseudo:
31080b57cec5SDimitry Andric     case ARM::VST2LNd8Pseudo_UPD:
31090b57cec5SDimitry Andric     case ARM::VST2LNd16Pseudo_UPD:
31100b57cec5SDimitry Andric     case ARM::VST2LNd32Pseudo_UPD:
31110b57cec5SDimitry Andric     case ARM::VST2LNq16Pseudo_UPD:
31120b57cec5SDimitry Andric     case ARM::VST2LNq32Pseudo_UPD:
31130b57cec5SDimitry Andric     case ARM::VST3LNd8Pseudo:
31140b57cec5SDimitry Andric     case ARM::VST3LNd16Pseudo:
31150b57cec5SDimitry Andric     case ARM::VST3LNd32Pseudo:
31160b57cec5SDimitry Andric     case ARM::VST3LNq16Pseudo:
31170b57cec5SDimitry Andric     case ARM::VST3LNq32Pseudo:
31180b57cec5SDimitry Andric     case ARM::VST3LNd8Pseudo_UPD:
31190b57cec5SDimitry Andric     case ARM::VST3LNd16Pseudo_UPD:
31200b57cec5SDimitry Andric     case ARM::VST3LNd32Pseudo_UPD:
31210b57cec5SDimitry Andric     case ARM::VST3LNq16Pseudo_UPD:
31220b57cec5SDimitry Andric     case ARM::VST3LNq32Pseudo_UPD:
31230b57cec5SDimitry Andric     case ARM::VST4LNd8Pseudo:
31240b57cec5SDimitry Andric     case ARM::VST4LNd16Pseudo:
31250b57cec5SDimitry Andric     case ARM::VST4LNd32Pseudo:
31260b57cec5SDimitry Andric     case ARM::VST4LNq16Pseudo:
31270b57cec5SDimitry Andric     case ARM::VST4LNq32Pseudo:
31280b57cec5SDimitry Andric     case ARM::VST4LNd8Pseudo_UPD:
31290b57cec5SDimitry Andric     case ARM::VST4LNd16Pseudo_UPD:
31300b57cec5SDimitry Andric     case ARM::VST4LNd32Pseudo_UPD:
31310b57cec5SDimitry Andric     case ARM::VST4LNq16Pseudo_UPD:
31320b57cec5SDimitry Andric     case ARM::VST4LNq32Pseudo_UPD:
31330b57cec5SDimitry Andric       ExpandLaneOp(MBBI);
31340b57cec5SDimitry Andric       return true;
31350b57cec5SDimitry Andric 
31360b57cec5SDimitry Andric     case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true;
31370b57cec5SDimitry Andric     case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;
31380b57cec5SDimitry Andric     case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;
31390b57cec5SDimitry Andric     case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;
31400b57cec5SDimitry Andric 
3141349cc55cSDimitry Andric     case ARM::MQQPRLoad:
3142349cc55cSDimitry Andric     case ARM::MQQPRStore:
3143349cc55cSDimitry Andric     case ARM::MQQQQPRLoad:
3144349cc55cSDimitry Andric     case ARM::MQQQQPRStore:
3145349cc55cSDimitry Andric       ExpandMQQPRLoadStore(MBBI);
3146349cc55cSDimitry Andric       return true;
3147349cc55cSDimitry Andric 
3148fe6060f1SDimitry Andric     case ARM::tCMP_SWAP_8:
3149fe6060f1SDimitry Andric       assert(STI->isThumb());
3150fe6060f1SDimitry Andric       return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB, ARM::tUXTB,
3151fe6060f1SDimitry Andric                             NextMBBI);
3152fe6060f1SDimitry Andric     case ARM::tCMP_SWAP_16:
3153fe6060f1SDimitry Andric       assert(STI->isThumb());
3154fe6060f1SDimitry Andric       return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH, ARM::tUXTH,
3155fe6060f1SDimitry Andric                             NextMBBI);
3156fcaf7f86SDimitry Andric     case ARM::tCMP_SWAP_32:
3157fcaf7f86SDimitry Andric       assert(STI->isThumb());
3158fcaf7f86SDimitry Andric       return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0, NextMBBI);
3159fe6060f1SDimitry Andric 
31600b57cec5SDimitry Andric     case ARM::CMP_SWAP_8:
3161fe6060f1SDimitry Andric       assert(!STI->isThumb());
3162fe6060f1SDimitry Andric       return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB, ARM::UXTB,
3163fe6060f1SDimitry Andric                             NextMBBI);
31640b57cec5SDimitry Andric     case ARM::CMP_SWAP_16:
3165fe6060f1SDimitry Andric       assert(!STI->isThumb());
3166fe6060f1SDimitry Andric       return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH, ARM::UXTH,
3167fe6060f1SDimitry Andric                             NextMBBI);
31680b57cec5SDimitry Andric     case ARM::CMP_SWAP_32:
3169fcaf7f86SDimitry Andric       assert(!STI->isThumb());
31700b57cec5SDimitry Andric       return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);
31710b57cec5SDimitry Andric 
31720b57cec5SDimitry Andric     case ARM::CMP_SWAP_64:
31730b57cec5SDimitry Andric       return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
31748bcb0991SDimitry Andric 
31758bcb0991SDimitry Andric     case ARM::tBL_PUSHLR:
31768bcb0991SDimitry Andric     case ARM::BL_PUSHLR: {
31778bcb0991SDimitry Andric       const bool Thumb = Opcode == ARM::tBL_PUSHLR;
31788bcb0991SDimitry Andric       Register Reg = MI.getOperand(0).getReg();
31798bcb0991SDimitry Andric       assert(Reg == ARM::LR && "expect LR register!");
31808bcb0991SDimitry Andric       MachineInstrBuilder MIB;
31818bcb0991SDimitry Andric       if (Thumb) {
31828bcb0991SDimitry Andric         // push {lr}
31838bcb0991SDimitry Andric         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPUSH))
31848bcb0991SDimitry Andric             .add(predOps(ARMCC::AL))
31858bcb0991SDimitry Andric             .addReg(Reg);
31868bcb0991SDimitry Andric 
31878bcb0991SDimitry Andric         // bl __gnu_mcount_nc
31888bcb0991SDimitry Andric         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tBL));
31898bcb0991SDimitry Andric       } else {
31908bcb0991SDimitry Andric         // stmdb   sp!, {lr}
31918bcb0991SDimitry Andric         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::STMDB_UPD))
31928bcb0991SDimitry Andric             .addReg(ARM::SP, RegState::Define)
31938bcb0991SDimitry Andric             .addReg(ARM::SP)
31948bcb0991SDimitry Andric             .add(predOps(ARMCC::AL))
31958bcb0991SDimitry Andric             .addReg(Reg);
31968bcb0991SDimitry Andric 
31978bcb0991SDimitry Andric         // bl __gnu_mcount_nc
31988bcb0991SDimitry Andric         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::BL));
31998bcb0991SDimitry Andric       }
32008bcb0991SDimitry Andric       MIB.cloneMemRefs(MI);
32014824e7fdSDimitry Andric       for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
32024824e7fdSDimitry Andric         MIB.add(MO);
32038bcb0991SDimitry Andric       MI.eraseFromParent();
32048bcb0991SDimitry Andric       return true;
32058bcb0991SDimitry Andric     }
32060eae32dcSDimitry Andric     case ARM::t2CALL_BTI: {
32070eae32dcSDimitry Andric       MachineFunction &MF = *MI.getMF();
32080eae32dcSDimitry Andric       MachineInstrBuilder MIB =
32090eae32dcSDimitry Andric           BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::tBL));
32100eae32dcSDimitry Andric       MIB.cloneMemRefs(MI);
32110eae32dcSDimitry Andric       for (unsigned i = 0; i < MI.getNumOperands(); ++i)
32120eae32dcSDimitry Andric         MIB.add(MI.getOperand(i));
32130eae32dcSDimitry Andric       if (MI.isCandidateForCallSiteEntry())
32140eae32dcSDimitry Andric         MF.moveCallSiteInfo(&MI, MIB.getInstr());
32150eae32dcSDimitry Andric       MIBundleBuilder Bundler(MBB, MI);
32160eae32dcSDimitry Andric       Bundler.append(MIB);
32170eae32dcSDimitry Andric       Bundler.append(BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::t2BTI)));
32180eae32dcSDimitry Andric       finalizeBundle(MBB, Bundler.begin(), Bundler.end());
32190eae32dcSDimitry Andric       MI.eraseFromParent();
32200eae32dcSDimitry Andric       return true;
32210eae32dcSDimitry Andric     }
32225ffd83dbSDimitry Andric     case ARM::LOADDUAL:
32235ffd83dbSDimitry Andric     case ARM::STOREDUAL: {
32245ffd83dbSDimitry Andric       Register PairReg = MI.getOperand(0).getReg();
32255ffd83dbSDimitry Andric 
32265ffd83dbSDimitry Andric       MachineInstrBuilder MIB =
32275ffd83dbSDimitry Andric           BuildMI(MBB, MBBI, MI.getDebugLoc(),
32285ffd83dbSDimitry Andric                   TII->get(Opcode == ARM::LOADDUAL ? ARM::LDRD : ARM::STRD))
32295ffd83dbSDimitry Andric               .addReg(TRI->getSubReg(PairReg, ARM::gsub_0),
32305ffd83dbSDimitry Andric                       Opcode == ARM::LOADDUAL ? RegState::Define : 0)
32315ffd83dbSDimitry Andric               .addReg(TRI->getSubReg(PairReg, ARM::gsub_1),
32325ffd83dbSDimitry Andric                       Opcode == ARM::LOADDUAL ? RegState::Define : 0);
32334824e7fdSDimitry Andric       for (const MachineOperand &MO : llvm::drop_begin(MI.operands()))
32344824e7fdSDimitry Andric         MIB.add(MO);
32355ffd83dbSDimitry Andric       MIB.add(predOps(ARMCC::AL));
32365ffd83dbSDimitry Andric       MIB.cloneMemRefs(MI);
32375ffd83dbSDimitry Andric       MI.eraseFromParent();
32385ffd83dbSDimitry Andric       return true;
32395ffd83dbSDimitry Andric     }
32400b57cec5SDimitry Andric   }
32410b57cec5SDimitry Andric }
32420b57cec5SDimitry Andric 
32430b57cec5SDimitry Andric bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
32440b57cec5SDimitry Andric   bool Modified = false;
32450b57cec5SDimitry Andric 
32460b57cec5SDimitry Andric   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
32470b57cec5SDimitry Andric   while (MBBI != E) {
32480b57cec5SDimitry Andric     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
32490b57cec5SDimitry Andric     Modified |= ExpandMI(MBB, MBBI, NMBBI);
32500b57cec5SDimitry Andric     MBBI = NMBBI;
32510b57cec5SDimitry Andric   }
32520b57cec5SDimitry Andric 
32530b57cec5SDimitry Andric   return Modified;
32540b57cec5SDimitry Andric }
32550b57cec5SDimitry Andric 
32560b57cec5SDimitry Andric bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
325781ad6265SDimitry Andric   STI = &MF.getSubtarget<ARMSubtarget>();
32580b57cec5SDimitry Andric   TII = STI->getInstrInfo();
32590b57cec5SDimitry Andric   TRI = STI->getRegisterInfo();
32600b57cec5SDimitry Andric   AFI = MF.getInfo<ARMFunctionInfo>();
32610b57cec5SDimitry Andric 
32620b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "********** ARM EXPAND PSEUDO INSTRUCTIONS **********\n"
32630b57cec5SDimitry Andric                     << "********** Function: " << MF.getName() << '\n');
32640b57cec5SDimitry Andric 
32650b57cec5SDimitry Andric   bool Modified = false;
32660b57cec5SDimitry Andric   for (MachineBasicBlock &MBB : MF)
32670b57cec5SDimitry Andric     Modified |= ExpandMBB(MBB);
32680b57cec5SDimitry Andric   if (VerifyARMPseudo)
32690b57cec5SDimitry Andric     MF.verify(this, "After expanding ARM pseudo instructions.");
32700b57cec5SDimitry Andric 
32710b57cec5SDimitry Andric   LLVM_DEBUG(dbgs() << "***************************************************\n");
32720b57cec5SDimitry Andric   return Modified;
32730b57cec5SDimitry Andric }
32740b57cec5SDimitry Andric 
32750b57cec5SDimitry Andric /// createARMExpandPseudoPass - returns an instance of the pseudo instruction
32760b57cec5SDimitry Andric /// expansion pass.
32770b57cec5SDimitry Andric FunctionPass *llvm::createARMExpandPseudoPass() {
32780b57cec5SDimitry Andric   return new ARMExpandPseudo();
32790b57cec5SDimitry Andric }
3280