xref: /freebsd-src/contrib/llvm-project/llvm/lib/Target/X86/X86MCInstLower.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
10b57cec5SDimitry Andric //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file contains code to lower X86 MachineInstrs to their corresponding
100b57cec5SDimitry Andric // MCInst records.
110b57cec5SDimitry Andric //
120b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
130b57cec5SDimitry Andric 
140b57cec5SDimitry Andric #include "MCTargetDesc/X86ATTInstPrinter.h"
150b57cec5SDimitry Andric #include "MCTargetDesc/X86BaseInfo.h"
1606c3fb27SDimitry Andric #include "MCTargetDesc/X86EncodingOptimization.h"
170b57cec5SDimitry Andric #include "MCTargetDesc/X86InstComments.h"
185ffd83dbSDimitry Andric #include "MCTargetDesc/X86ShuffleDecode.h"
190b57cec5SDimitry Andric #include "MCTargetDesc/X86TargetStreamer.h"
200b57cec5SDimitry Andric #include "X86AsmPrinter.h"
2106c3fb27SDimitry Andric #include "X86MachineFunctionInfo.h"
220b57cec5SDimitry Andric #include "X86RegisterInfo.h"
230b57cec5SDimitry Andric #include "X86ShuffleDecodeConstantPool.h"
245ffd83dbSDimitry Andric #include "X86Subtarget.h"
25*0fca6ea1SDimitry Andric #include "llvm/ADT/STLExtras.h"
260b57cec5SDimitry Andric #include "llvm/ADT/SmallString.h"
2706c3fb27SDimitry Andric #include "llvm/ADT/StringExtras.h"
28*0fca6ea1SDimitry Andric #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
290b57cec5SDimitry Andric #include "llvm/CodeGen/MachineConstantPool.h"
300b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h"
310b57cec5SDimitry Andric #include "llvm/CodeGen/MachineModuleInfoImpls.h"
320b57cec5SDimitry Andric #include "llvm/CodeGen/MachineOperand.h"
330b57cec5SDimitry Andric #include "llvm/CodeGen/StackMaps.h"
340b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
350b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
360b57cec5SDimitry Andric #include "llvm/IR/Mangler.h"
370b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h"
380b57cec5SDimitry Andric #include "llvm/MC/MCCodeEmitter.h"
390b57cec5SDimitry Andric #include "llvm/MC/MCContext.h"
400b57cec5SDimitry Andric #include "llvm/MC/MCExpr.h"
410b57cec5SDimitry Andric #include "llvm/MC/MCFixup.h"
420b57cec5SDimitry Andric #include "llvm/MC/MCInst.h"
430b57cec5SDimitry Andric #include "llvm/MC/MCInstBuilder.h"
440b57cec5SDimitry Andric #include "llvm/MC/MCSection.h"
450b57cec5SDimitry Andric #include "llvm/MC/MCSectionELF.h"
460b57cec5SDimitry Andric #include "llvm/MC/MCStreamer.h"
470b57cec5SDimitry Andric #include "llvm/MC/MCSymbol.h"
480b57cec5SDimitry Andric #include "llvm/MC/MCSymbolELF.h"
49349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h"
500b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h"
515ffd83dbSDimitry Andric #include "llvm/Target/TargetMachine.h"
52349cc55cSDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
53349cc55cSDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
540eae32dcSDimitry Andric #include <string>
550b57cec5SDimitry Andric 
560b57cec5SDimitry Andric using namespace llvm;
570b57cec5SDimitry Andric 
58*0fca6ea1SDimitry Andric static cl::opt<bool> EnableBranchHint("enable-branch-hint",
59*0fca6ea1SDimitry Andric                                       cl::desc("Enable branch hint."),
60*0fca6ea1SDimitry Andric                                       cl::init(false), cl::Hidden);
61*0fca6ea1SDimitry Andric static cl::opt<unsigned> BranchHintProbabilityThreshold(
62*0fca6ea1SDimitry Andric     "branch-hint-probability-threshold",
63*0fca6ea1SDimitry Andric     cl::desc("The probability threshold of enabling branch hint."),
64*0fca6ea1SDimitry Andric     cl::init(50), cl::Hidden);
65*0fca6ea1SDimitry Andric 
660b57cec5SDimitry Andric namespace {
670b57cec5SDimitry Andric 
680b57cec5SDimitry Andric /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
690b57cec5SDimitry Andric class X86MCInstLower {
700b57cec5SDimitry Andric   MCContext &Ctx;
710b57cec5SDimitry Andric   const MachineFunction &MF;
720b57cec5SDimitry Andric   const TargetMachine &TM;
730b57cec5SDimitry Andric   const MCAsmInfo &MAI;
740b57cec5SDimitry Andric   X86AsmPrinter &AsmPrinter;
750b57cec5SDimitry Andric 
760b57cec5SDimitry Andric public:
770b57cec5SDimitry Andric   X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
780b57cec5SDimitry Andric 
79*0fca6ea1SDimitry Andric   MCOperand LowerMachineOperand(const MachineInstr *MI,
800b57cec5SDimitry Andric                                 const MachineOperand &MO) const;
810b57cec5SDimitry Andric   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
820b57cec5SDimitry Andric 
830b57cec5SDimitry Andric   MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
840b57cec5SDimitry Andric   MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
850b57cec5SDimitry Andric 
860b57cec5SDimitry Andric private:
870b57cec5SDimitry Andric   MachineModuleInfoMachO &getMachOMMI() const;
880b57cec5SDimitry Andric };
890b57cec5SDimitry Andric 
900b57cec5SDimitry Andric } // end anonymous namespace
910b57cec5SDimitry Andric 
925ffd83dbSDimitry Andric /// A RAII helper which defines a region of instructions which can't have
935ffd83dbSDimitry Andric /// padding added between them for correctness.
945ffd83dbSDimitry Andric struct NoAutoPaddingScope {
955ffd83dbSDimitry Andric   MCStreamer &OS;
965ffd83dbSDimitry Andric   const bool OldAllowAutoPadding;
975ffd83dbSDimitry Andric   NoAutoPaddingScope(MCStreamer &OS)
985ffd83dbSDimitry Andric       : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
995ffd83dbSDimitry Andric     changeAndComment(false);
1005ffd83dbSDimitry Andric   }
1015ffd83dbSDimitry Andric   ~NoAutoPaddingScope() { changeAndComment(OldAllowAutoPadding); }
1025ffd83dbSDimitry Andric   void changeAndComment(bool b) {
1035ffd83dbSDimitry Andric     if (b == OS.getAllowAutoPadding())
1045ffd83dbSDimitry Andric       return;
1055ffd83dbSDimitry Andric     OS.setAllowAutoPadding(b);
1065ffd83dbSDimitry Andric     if (b)
1075ffd83dbSDimitry Andric       OS.emitRawComment("autopadding");
1085ffd83dbSDimitry Andric     else
1095ffd83dbSDimitry Andric       OS.emitRawComment("noautopadding");
1105ffd83dbSDimitry Andric   }
1115ffd83dbSDimitry Andric };
1125ffd83dbSDimitry Andric 
1130b57cec5SDimitry Andric // Emit a minimal sequence of nops spanning NumBytes bytes.
1145ffd83dbSDimitry Andric static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
1155ffd83dbSDimitry Andric                         const X86Subtarget *Subtarget);
1160b57cec5SDimitry Andric 
1170b57cec5SDimitry Andric void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst,
1180b57cec5SDimitry Andric                                                  const MCSubtargetInfo &STI,
1190b57cec5SDimitry Andric                                                  MCCodeEmitter *CodeEmitter) {
1200b57cec5SDimitry Andric   if (InShadow) {
1210b57cec5SDimitry Andric     SmallString<256> Code;
1220b57cec5SDimitry Andric     SmallVector<MCFixup, 4> Fixups;
12306c3fb27SDimitry Andric     CodeEmitter->encodeInstruction(Inst, Code, Fixups, STI);
1240b57cec5SDimitry Andric     CurrentShadowSize += Code.size();
1250b57cec5SDimitry Andric     if (CurrentShadowSize >= RequiredShadowSize)
1260b57cec5SDimitry Andric       InShadow = false; // The shadow is big enough. Stop counting.
1270b57cec5SDimitry Andric   }
1280b57cec5SDimitry Andric }
1290b57cec5SDimitry Andric 
1300b57cec5SDimitry Andric void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
1310b57cec5SDimitry Andric     MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
1320b57cec5SDimitry Andric   if (InShadow && CurrentShadowSize < RequiredShadowSize) {
1330b57cec5SDimitry Andric     InShadow = false;
1345ffd83dbSDimitry Andric     emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,
1355ffd83dbSDimitry Andric                 &MF->getSubtarget<X86Subtarget>());
1360b57cec5SDimitry Andric   }
1370b57cec5SDimitry Andric }
1380b57cec5SDimitry Andric 
1390b57cec5SDimitry Andric void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
1405ffd83dbSDimitry Andric   OutStreamer->emitInstruction(Inst, getSubtargetInfo());
1410b57cec5SDimitry Andric   SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
1420b57cec5SDimitry Andric }
1430b57cec5SDimitry Andric 
1440b57cec5SDimitry Andric X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
1450b57cec5SDimitry Andric                                X86AsmPrinter &asmprinter)
1460b57cec5SDimitry Andric     : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()),
1470b57cec5SDimitry Andric       AsmPrinter(asmprinter) {}
1480b57cec5SDimitry Andric 
1490b57cec5SDimitry Andric MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
150*0fca6ea1SDimitry Andric   return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();
1510b57cec5SDimitry Andric }
1520b57cec5SDimitry Andric 
1530b57cec5SDimitry Andric /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
1540b57cec5SDimitry Andric /// operand to an MCSymbol.
1550b57cec5SDimitry Andric MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
1565ffd83dbSDimitry Andric   const Triple &TT = TM.getTargetTriple();
1575ffd83dbSDimitry Andric   if (MO.isGlobal() && TT.isOSBinFormatELF())
1585ffd83dbSDimitry Andric     return AsmPrinter.getSymbolPreferLocal(*MO.getGlobal());
1595ffd83dbSDimitry Andric 
1600b57cec5SDimitry Andric   const DataLayout &DL = MF.getDataLayout();
1610b57cec5SDimitry Andric   assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
1620b57cec5SDimitry Andric          "Isn't a symbol reference");
1630b57cec5SDimitry Andric 
1640b57cec5SDimitry Andric   MCSymbol *Sym = nullptr;
1650b57cec5SDimitry Andric   SmallString<128> Name;
1660b57cec5SDimitry Andric   StringRef Suffix;
1670b57cec5SDimitry Andric 
1680b57cec5SDimitry Andric   switch (MO.getTargetFlags()) {
1690b57cec5SDimitry Andric   case X86II::MO_DLLIMPORT:
1700b57cec5SDimitry Andric     // Handle dllimport linkage.
1710b57cec5SDimitry Andric     Name += "__imp_";
1720b57cec5SDimitry Andric     break;
1730b57cec5SDimitry Andric   case X86II::MO_COFFSTUB:
1740b57cec5SDimitry Andric     Name += ".refptr.";
1750b57cec5SDimitry Andric     break;
1760b57cec5SDimitry Andric   case X86II::MO_DARWIN_NONLAZY:
1770b57cec5SDimitry Andric   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
1780b57cec5SDimitry Andric     Suffix = "$non_lazy_ptr";
1790b57cec5SDimitry Andric     break;
1800b57cec5SDimitry Andric   }
1810b57cec5SDimitry Andric 
1820b57cec5SDimitry Andric   if (!Suffix.empty())
1830b57cec5SDimitry Andric     Name += DL.getPrivateGlobalPrefix();
1840b57cec5SDimitry Andric 
1850b57cec5SDimitry Andric   if (MO.isGlobal()) {
1860b57cec5SDimitry Andric     const GlobalValue *GV = MO.getGlobal();
1870b57cec5SDimitry Andric     AsmPrinter.getNameWithPrefix(Name, GV);
1880b57cec5SDimitry Andric   } else if (MO.isSymbol()) {
1890b57cec5SDimitry Andric     Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);
1900b57cec5SDimitry Andric   } else if (MO.isMBB()) {
1910b57cec5SDimitry Andric     assert(Suffix.empty());
1920b57cec5SDimitry Andric     Sym = MO.getMBB()->getSymbol();
1930b57cec5SDimitry Andric   }
1940b57cec5SDimitry Andric 
1950b57cec5SDimitry Andric   Name += Suffix;
1960b57cec5SDimitry Andric   if (!Sym)
1970b57cec5SDimitry Andric     Sym = Ctx.getOrCreateSymbol(Name);
1980b57cec5SDimitry Andric 
1990b57cec5SDimitry Andric   // If the target flags on the operand changes the name of the symbol, do that
2000b57cec5SDimitry Andric   // before we return the symbol.
2010b57cec5SDimitry Andric   switch (MO.getTargetFlags()) {
2020b57cec5SDimitry Andric   default:
2030b57cec5SDimitry Andric     break;
2040b57cec5SDimitry Andric   case X86II::MO_COFFSTUB: {
2050b57cec5SDimitry Andric     MachineModuleInfoCOFF &MMICOFF =
206*0fca6ea1SDimitry Andric         AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();
2070b57cec5SDimitry Andric     MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
2080b57cec5SDimitry Andric     if (!StubSym.getPointer()) {
2090b57cec5SDimitry Andric       assert(MO.isGlobal() && "Extern symbol not handled yet");
2100b57cec5SDimitry Andric       StubSym = MachineModuleInfoImpl::StubValueTy(
2110b57cec5SDimitry Andric           AsmPrinter.getSymbol(MO.getGlobal()), true);
2120b57cec5SDimitry Andric     }
2130b57cec5SDimitry Andric     break;
2140b57cec5SDimitry Andric   }
2150b57cec5SDimitry Andric   case X86II::MO_DARWIN_NONLAZY:
2160b57cec5SDimitry Andric   case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
2170b57cec5SDimitry Andric     MachineModuleInfoImpl::StubValueTy &StubSym =
2180b57cec5SDimitry Andric         getMachOMMI().getGVStubEntry(Sym);
2190b57cec5SDimitry Andric     if (!StubSym.getPointer()) {
2200b57cec5SDimitry Andric       assert(MO.isGlobal() && "Extern symbol not handled yet");
2210b57cec5SDimitry Andric       StubSym = MachineModuleInfoImpl::StubValueTy(
2220b57cec5SDimitry Andric           AsmPrinter.getSymbol(MO.getGlobal()),
2230b57cec5SDimitry Andric           !MO.getGlobal()->hasInternalLinkage());
2240b57cec5SDimitry Andric     }
2250b57cec5SDimitry Andric     break;
2260b57cec5SDimitry Andric   }
2270b57cec5SDimitry Andric   }
2280b57cec5SDimitry Andric 
2290b57cec5SDimitry Andric   return Sym;
2300b57cec5SDimitry Andric }
2310b57cec5SDimitry Andric 
2320b57cec5SDimitry Andric MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
2330b57cec5SDimitry Andric                                              MCSymbol *Sym) const {
2340b57cec5SDimitry Andric   // FIXME: We would like an efficient form for this, so we don't have to do a
2350b57cec5SDimitry Andric   // lot of extra uniquing.
2360b57cec5SDimitry Andric   const MCExpr *Expr = nullptr;
2370b57cec5SDimitry Andric   MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None;
2380b57cec5SDimitry Andric 
2390b57cec5SDimitry Andric   switch (MO.getTargetFlags()) {
2400b57cec5SDimitry Andric   default:
2410b57cec5SDimitry Andric     llvm_unreachable("Unknown target flag on GV operand");
2420b57cec5SDimitry Andric   case X86II::MO_NO_FLAG: // No flag.
2430b57cec5SDimitry Andric   // These affect the name of the symbol, not any suffix.
2440b57cec5SDimitry Andric   case X86II::MO_DARWIN_NONLAZY:
2450b57cec5SDimitry Andric   case X86II::MO_DLLIMPORT:
2460b57cec5SDimitry Andric   case X86II::MO_COFFSTUB:
2470b57cec5SDimitry Andric     break;
2480b57cec5SDimitry Andric 
2490b57cec5SDimitry Andric   case X86II::MO_TLVP:
2500b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_TLVP;
2510b57cec5SDimitry Andric     break;
2520b57cec5SDimitry Andric   case X86II::MO_TLVP_PIC_BASE:
2530b57cec5SDimitry Andric     Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx);
2540b57cec5SDimitry Andric     // Subtract the pic base.
2550b57cec5SDimitry Andric     Expr = MCBinaryExpr::createSub(
2560b57cec5SDimitry Andric         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
2570b57cec5SDimitry Andric     break;
2580b57cec5SDimitry Andric   case X86II::MO_SECREL:
2590b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_SECREL;
2600b57cec5SDimitry Andric     break;
2610b57cec5SDimitry Andric   case X86II::MO_TLSGD:
2620b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_TLSGD;
2630b57cec5SDimitry Andric     break;
2640b57cec5SDimitry Andric   case X86II::MO_TLSLD:
2650b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_TLSLD;
2660b57cec5SDimitry Andric     break;
2670b57cec5SDimitry Andric   case X86II::MO_TLSLDM:
2680b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_TLSLDM;
2690b57cec5SDimitry Andric     break;
2700b57cec5SDimitry Andric   case X86II::MO_GOTTPOFF:
2710b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_GOTTPOFF;
2720b57cec5SDimitry Andric     break;
2730b57cec5SDimitry Andric   case X86II::MO_INDNTPOFF:
2740b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_INDNTPOFF;
2750b57cec5SDimitry Andric     break;
2760b57cec5SDimitry Andric   case X86II::MO_TPOFF:
2770b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_TPOFF;
2780b57cec5SDimitry Andric     break;
2790b57cec5SDimitry Andric   case X86II::MO_DTPOFF:
2800b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_DTPOFF;
2810b57cec5SDimitry Andric     break;
2820b57cec5SDimitry Andric   case X86II::MO_NTPOFF:
2830b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_NTPOFF;
2840b57cec5SDimitry Andric     break;
2850b57cec5SDimitry Andric   case X86II::MO_GOTNTPOFF:
2860b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_GOTNTPOFF;
2870b57cec5SDimitry Andric     break;
2880b57cec5SDimitry Andric   case X86II::MO_GOTPCREL:
2890b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_GOTPCREL;
2900b57cec5SDimitry Andric     break;
291349cc55cSDimitry Andric   case X86II::MO_GOTPCREL_NORELAX:
292349cc55cSDimitry Andric     RefKind = MCSymbolRefExpr::VK_GOTPCREL_NORELAX;
293349cc55cSDimitry Andric     break;
2940b57cec5SDimitry Andric   case X86II::MO_GOT:
2950b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_GOT;
2960b57cec5SDimitry Andric     break;
2970b57cec5SDimitry Andric   case X86II::MO_GOTOFF:
2980b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_GOTOFF;
2990b57cec5SDimitry Andric     break;
3000b57cec5SDimitry Andric   case X86II::MO_PLT:
3010b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_PLT;
3020b57cec5SDimitry Andric     break;
3030b57cec5SDimitry Andric   case X86II::MO_ABS8:
3040b57cec5SDimitry Andric     RefKind = MCSymbolRefExpr::VK_X86_ABS8;
3050b57cec5SDimitry Andric     break;
3060b57cec5SDimitry Andric   case X86II::MO_PIC_BASE_OFFSET:
3070b57cec5SDimitry Andric   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
3080b57cec5SDimitry Andric     Expr = MCSymbolRefExpr::create(Sym, Ctx);
3090b57cec5SDimitry Andric     // Subtract the pic base.
3100b57cec5SDimitry Andric     Expr = MCBinaryExpr::createSub(
3110b57cec5SDimitry Andric         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
3120b57cec5SDimitry Andric     if (MO.isJTI()) {
3130b57cec5SDimitry Andric       assert(MAI.doesSetDirectiveSuppressReloc());
3140b57cec5SDimitry Andric       // If .set directive is supported, use it to reduce the number of
3150b57cec5SDimitry Andric       // relocations the assembler will generate for differences between
3160b57cec5SDimitry Andric       // local labels. This is only safe when the symbols are in the same
3170b57cec5SDimitry Andric       // section so we are restricting it to jumptable references.
3180b57cec5SDimitry Andric       MCSymbol *Label = Ctx.createTempSymbol();
3195ffd83dbSDimitry Andric       AsmPrinter.OutStreamer->emitAssignment(Label, Expr);
3200b57cec5SDimitry Andric       Expr = MCSymbolRefExpr::create(Label, Ctx);
3210b57cec5SDimitry Andric     }
3220b57cec5SDimitry Andric     break;
3230b57cec5SDimitry Andric   }
3240b57cec5SDimitry Andric 
3250b57cec5SDimitry Andric   if (!Expr)
3260b57cec5SDimitry Andric     Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
3270b57cec5SDimitry Andric 
3280b57cec5SDimitry Andric   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
3290b57cec5SDimitry Andric     Expr = MCBinaryExpr::createAdd(
3300b57cec5SDimitry Andric         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
3310b57cec5SDimitry Andric   return MCOperand::createExpr(Expr);
3320b57cec5SDimitry Andric }
3330b57cec5SDimitry Andric 
3340b57cec5SDimitry Andric static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
335349cc55cSDimitry Andric   return Subtarget.is64Bit() ? X86::RET64 : X86::RET32;
3360b57cec5SDimitry Andric }
3370b57cec5SDimitry Andric 
338*0fca6ea1SDimitry Andric MCOperand X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
3390b57cec5SDimitry Andric                                               const MachineOperand &MO) const {
3400b57cec5SDimitry Andric   switch (MO.getType()) {
3410b57cec5SDimitry Andric   default:
3420b57cec5SDimitry Andric     MI->print(errs());
3430b57cec5SDimitry Andric     llvm_unreachable("unknown operand type");
3440b57cec5SDimitry Andric   case MachineOperand::MO_Register:
3450b57cec5SDimitry Andric     // Ignore all implicit register operands.
3460b57cec5SDimitry Andric     if (MO.isImplicit())
347*0fca6ea1SDimitry Andric       return MCOperand();
3480b57cec5SDimitry Andric     return MCOperand::createReg(MO.getReg());
3490b57cec5SDimitry Andric   case MachineOperand::MO_Immediate:
3500b57cec5SDimitry Andric     return MCOperand::createImm(MO.getImm());
3510b57cec5SDimitry Andric   case MachineOperand::MO_MachineBasicBlock:
3520b57cec5SDimitry Andric   case MachineOperand::MO_GlobalAddress:
3530b57cec5SDimitry Andric   case MachineOperand::MO_ExternalSymbol:
3540b57cec5SDimitry Andric     return LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
3550b57cec5SDimitry Andric   case MachineOperand::MO_MCSymbol:
3560b57cec5SDimitry Andric     return LowerSymbolOperand(MO, MO.getMCSymbol());
3570b57cec5SDimitry Andric   case MachineOperand::MO_JumpTableIndex:
3580b57cec5SDimitry Andric     return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));
3590b57cec5SDimitry Andric   case MachineOperand::MO_ConstantPoolIndex:
3600b57cec5SDimitry Andric     return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));
3610b57cec5SDimitry Andric   case MachineOperand::MO_BlockAddress:
3620b57cec5SDimitry Andric     return LowerSymbolOperand(
3630b57cec5SDimitry Andric         MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));
3640b57cec5SDimitry Andric   case MachineOperand::MO_RegisterMask:
3650b57cec5SDimitry Andric     // Ignore call clobbers.
366*0fca6ea1SDimitry Andric     return MCOperand();
3670b57cec5SDimitry Andric   }
3680b57cec5SDimitry Andric }
3690b57cec5SDimitry Andric 
3708bcb0991SDimitry Andric // Replace TAILJMP opcodes with their equivalent opcodes that have encoding
3718bcb0991SDimitry Andric // information.
3728bcb0991SDimitry Andric static unsigned convertTailJumpOpcode(unsigned Opcode) {
3738bcb0991SDimitry Andric   switch (Opcode) {
3748bcb0991SDimitry Andric   case X86::TAILJMPr:
3758bcb0991SDimitry Andric     Opcode = X86::JMP32r;
3768bcb0991SDimitry Andric     break;
3778bcb0991SDimitry Andric   case X86::TAILJMPm:
3788bcb0991SDimitry Andric     Opcode = X86::JMP32m;
3798bcb0991SDimitry Andric     break;
3808bcb0991SDimitry Andric   case X86::TAILJMPr64:
3818bcb0991SDimitry Andric     Opcode = X86::JMP64r;
3828bcb0991SDimitry Andric     break;
3838bcb0991SDimitry Andric   case X86::TAILJMPm64:
3848bcb0991SDimitry Andric     Opcode = X86::JMP64m;
3858bcb0991SDimitry Andric     break;
3868bcb0991SDimitry Andric   case X86::TAILJMPr64_REX:
3878bcb0991SDimitry Andric     Opcode = X86::JMP64r_REX;
3888bcb0991SDimitry Andric     break;
3898bcb0991SDimitry Andric   case X86::TAILJMPm64_REX:
3908bcb0991SDimitry Andric     Opcode = X86::JMP64m_REX;
3918bcb0991SDimitry Andric     break;
3928bcb0991SDimitry Andric   case X86::TAILJMPd:
3938bcb0991SDimitry Andric   case X86::TAILJMPd64:
3948bcb0991SDimitry Andric     Opcode = X86::JMP_1;
3958bcb0991SDimitry Andric     break;
3968bcb0991SDimitry Andric   case X86::TAILJMPd_CC:
3978bcb0991SDimitry Andric   case X86::TAILJMPd64_CC:
3988bcb0991SDimitry Andric     Opcode = X86::JCC_1;
3998bcb0991SDimitry Andric     break;
4008bcb0991SDimitry Andric   }
4018bcb0991SDimitry Andric 
4028bcb0991SDimitry Andric   return Opcode;
4038bcb0991SDimitry Andric }
4048bcb0991SDimitry Andric 
4050b57cec5SDimitry Andric void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
4060b57cec5SDimitry Andric   OutMI.setOpcode(MI->getOpcode());
4070b57cec5SDimitry Andric 
4080b57cec5SDimitry Andric   for (const MachineOperand &MO : MI->operands())
409*0fca6ea1SDimitry Andric     if (auto Op = LowerMachineOperand(MI, MO); Op.isValid())
410*0fca6ea1SDimitry Andric       OutMI.addOperand(Op);
4110b57cec5SDimitry Andric 
41206c3fb27SDimitry Andric   bool In64BitMode = AsmPrinter.getSubtarget().is64Bit();
41306c3fb27SDimitry Andric   if (X86::optimizeInstFromVEX3ToVEX2(OutMI, MI->getDesc()) ||
41406c3fb27SDimitry Andric       X86::optimizeShiftRotateWithImmediateOne(OutMI) ||
41506c3fb27SDimitry Andric       X86::optimizeVPCMPWithImmediateOneOrSix(OutMI) ||
41606c3fb27SDimitry Andric       X86::optimizeMOVSX(OutMI) || X86::optimizeINCDEC(OutMI, In64BitMode) ||
41706c3fb27SDimitry Andric       X86::optimizeMOV(OutMI, In64BitMode) ||
41806c3fb27SDimitry Andric       X86::optimizeToFixedRegisterOrShortImmediateForm(OutMI))
41906c3fb27SDimitry Andric     return;
42006c3fb27SDimitry Andric 
4210b57cec5SDimitry Andric   // Handle a few special cases to eliminate operand modifiers.
4220b57cec5SDimitry Andric   switch (OutMI.getOpcode()) {
4230b57cec5SDimitry Andric   case X86::LEA64_32r:
4240b57cec5SDimitry Andric   case X86::LEA64r:
4250b57cec5SDimitry Andric   case X86::LEA16r:
4260b57cec5SDimitry Andric   case X86::LEA32r:
4270b57cec5SDimitry Andric     // LEA should have a segment register, but it must be empty.
4280b57cec5SDimitry Andric     assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
4290b57cec5SDimitry Andric            "Unexpected # of LEA operands");
4300b57cec5SDimitry Andric     assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
4310b57cec5SDimitry Andric            "LEA has segment specified!");
4320b57cec5SDimitry Andric     break;
4335ffd83dbSDimitry Andric   case X86::MULX32Hrr:
4345ffd83dbSDimitry Andric   case X86::MULX32Hrm:
4355ffd83dbSDimitry Andric   case X86::MULX64Hrr:
4365ffd83dbSDimitry Andric   case X86::MULX64Hrm: {
4375ffd83dbSDimitry Andric     // Turn into regular MULX by duplicating the destination.
4385ffd83dbSDimitry Andric     unsigned NewOpc;
4395ffd83dbSDimitry Andric     switch (OutMI.getOpcode()) {
4405ffd83dbSDimitry Andric     default: llvm_unreachable("Invalid opcode");
4415ffd83dbSDimitry Andric     case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
4425ffd83dbSDimitry Andric     case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
4435ffd83dbSDimitry Andric     case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
4445ffd83dbSDimitry Andric     case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
4455ffd83dbSDimitry Andric     }
4465ffd83dbSDimitry Andric     OutMI.setOpcode(NewOpc);
4475ffd83dbSDimitry Andric     // Duplicate the destination.
4485ffd83dbSDimitry Andric     unsigned DestReg = OutMI.getOperand(0).getReg();
4495ffd83dbSDimitry Andric     OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg));
4505ffd83dbSDimitry Andric     break;
4515ffd83dbSDimitry Andric   }
4528bcb0991SDimitry Andric   // CALL64r, CALL64pcrel32 - These instructions used to have
4538bcb0991SDimitry Andric   // register inputs modeled as normal uses instead of implicit uses.  As such,
4548bcb0991SDimitry Andric   // they we used to truncate off all but the first operand (the callee). This
4558bcb0991SDimitry Andric   // issue seems to have been fixed at some point. This assert verifies that.
4568bcb0991SDimitry Andric   case X86::CALL64r:
4578bcb0991SDimitry Andric   case X86::CALL64pcrel32:
4588bcb0991SDimitry Andric     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
4598bcb0991SDimitry Andric     break;
4600b57cec5SDimitry Andric   case X86::EH_RETURN:
4610b57cec5SDimitry Andric   case X86::EH_RETURN64: {
4620b57cec5SDimitry Andric     OutMI = MCInst();
4630b57cec5SDimitry Andric     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
4640b57cec5SDimitry Andric     break;
4650b57cec5SDimitry Andric   }
4660b57cec5SDimitry Andric   case X86::CLEANUPRET: {
4670b57cec5SDimitry Andric     // Replace CLEANUPRET with the appropriate RET.
4680b57cec5SDimitry Andric     OutMI = MCInst();
4690b57cec5SDimitry Andric     OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
4700b57cec5SDimitry Andric     break;
4710b57cec5SDimitry Andric   }
4720b57cec5SDimitry Andric   case X86::CATCHRET: {
4730b57cec5SDimitry Andric     // Replace CATCHRET with the appropriate RET.
4740b57cec5SDimitry Andric     const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
47506c3fb27SDimitry Andric     unsigned ReturnReg = In64BitMode ? X86::RAX : X86::EAX;
4760b57cec5SDimitry Andric     OutMI = MCInst();
4770b57cec5SDimitry Andric     OutMI.setOpcode(getRetOpcode(Subtarget));
4780b57cec5SDimitry Andric     OutMI.addOperand(MCOperand::createReg(ReturnReg));
4790b57cec5SDimitry Andric     break;
4800b57cec5SDimitry Andric   }
4810b57cec5SDimitry Andric   // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
4820b57cec5SDimitry Andric   // instruction.
4830b57cec5SDimitry Andric   case X86::TAILJMPr:
4848bcb0991SDimitry Andric   case X86::TAILJMPr64:
4858bcb0991SDimitry Andric   case X86::TAILJMPr64_REX:
4860b57cec5SDimitry Andric   case X86::TAILJMPd:
4870b57cec5SDimitry Andric   case X86::TAILJMPd64:
4888bcb0991SDimitry Andric     assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
4898bcb0991SDimitry Andric     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
4900b57cec5SDimitry Andric     break;
4910b57cec5SDimitry Andric   case X86::TAILJMPd_CC:
4928bcb0991SDimitry Andric   case X86::TAILJMPd64_CC:
4938bcb0991SDimitry Andric     assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
4948bcb0991SDimitry Andric     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
4950b57cec5SDimitry Andric     break;
4968bcb0991SDimitry Andric   case X86::TAILJMPm:
4978bcb0991SDimitry Andric   case X86::TAILJMPm64:
4988bcb0991SDimitry Andric   case X86::TAILJMPm64_REX:
4998bcb0991SDimitry Andric     assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
5008bcb0991SDimitry Andric            "Unexpected number of operands!");
5018bcb0991SDimitry Andric     OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
5028bcb0991SDimitry Andric     break;
50381ad6265SDimitry Andric   case X86::MASKMOVDQU:
50481ad6265SDimitry Andric   case X86::VMASKMOVDQU:
50506c3fb27SDimitry Andric     if (In64BitMode)
50681ad6265SDimitry Andric       OutMI.setFlags(X86::IP_HAS_AD_SIZE);
50781ad6265SDimitry Andric     break;
50806c3fb27SDimitry Andric   case X86::BSF16rm:
50906c3fb27SDimitry Andric   case X86::BSF16rr:
51006c3fb27SDimitry Andric   case X86::BSF32rm:
51106c3fb27SDimitry Andric   case X86::BSF32rr:
51206c3fb27SDimitry Andric   case X86::BSF64rm:
51306c3fb27SDimitry Andric   case X86::BSF64rr: {
514bdd1243dSDimitry Andric     // Add an REP prefix to BSF instructions so that new processors can
515bdd1243dSDimitry Andric     // recognize as TZCNT, which has better performance than BSF.
516bdd1243dSDimitry Andric     // BSF and TZCNT have different interpretations on ZF bit. So make sure
517bdd1243dSDimitry Andric     // it won't be used later.
518*0fca6ea1SDimitry Andric     const MachineOperand *FlagDef =
519*0fca6ea1SDimitry Andric         MI->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
52006c3fb27SDimitry Andric     if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead())
521bdd1243dSDimitry Andric       OutMI.setFlags(X86::IP_HAS_REPEAT);
522480093f4SDimitry Andric     break;
523480093f4SDimitry Andric   }
52406c3fb27SDimitry Andric   default:
52506c3fb27SDimitry Andric     break;
5260b57cec5SDimitry Andric   }
5270b57cec5SDimitry Andric }
5280b57cec5SDimitry Andric 
5290b57cec5SDimitry Andric void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
5300b57cec5SDimitry Andric                                  const MachineInstr &MI) {
5315ffd83dbSDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
532*0fca6ea1SDimitry Andric   bool Is64Bits = getSubtarget().is64Bit();
533*0fca6ea1SDimitry Andric   bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();
5340b57cec5SDimitry Andric   MCContext &Ctx = OutStreamer->getContext();
5350b57cec5SDimitry Andric 
5360b57cec5SDimitry Andric   MCSymbolRefExpr::VariantKind SRVK;
5370b57cec5SDimitry Andric   switch (MI.getOpcode()) {
5380b57cec5SDimitry Andric   case X86::TLS_addr32:
5390b57cec5SDimitry Andric   case X86::TLS_addr64:
540e8d8bef9SDimitry Andric   case X86::TLS_addrX32:
5410b57cec5SDimitry Andric     SRVK = MCSymbolRefExpr::VK_TLSGD;
5420b57cec5SDimitry Andric     break;
5430b57cec5SDimitry Andric   case X86::TLS_base_addr32:
5440b57cec5SDimitry Andric     SRVK = MCSymbolRefExpr::VK_TLSLDM;
5450b57cec5SDimitry Andric     break;
5460b57cec5SDimitry Andric   case X86::TLS_base_addr64:
547e8d8bef9SDimitry Andric   case X86::TLS_base_addrX32:
5480b57cec5SDimitry Andric     SRVK = MCSymbolRefExpr::VK_TLSLD;
5490b57cec5SDimitry Andric     break;
550*0fca6ea1SDimitry Andric   case X86::TLS_desc32:
551*0fca6ea1SDimitry Andric   case X86::TLS_desc64:
552*0fca6ea1SDimitry Andric     SRVK = MCSymbolRefExpr::VK_TLSDESC;
553*0fca6ea1SDimitry Andric     break;
5540b57cec5SDimitry Andric   default:
5550b57cec5SDimitry Andric     llvm_unreachable("unexpected opcode");
5560b57cec5SDimitry Andric   }
5570b57cec5SDimitry Andric 
5580b57cec5SDimitry Andric   const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
5590b57cec5SDimitry Andric       MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx);
5600b57cec5SDimitry Andric 
561*0fca6ea1SDimitry Andric   // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD
5620b57cec5SDimitry Andric   // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
5630b57cec5SDimitry Andric   // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
5640b57cec5SDimitry Andric   // only using GOT when GOTPCRELX is enabled.
565*0fca6ea1SDimitry Andric   // TODO Delete the workaround when rustc no longer relies on the hack
5660b57cec5SDimitry Andric   bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
567*0fca6ea1SDimitry Andric                 Ctx.getTargetOptions()->X86RelaxRelocations;
5680b57cec5SDimitry Andric 
569*0fca6ea1SDimitry Andric   if (SRVK == MCSymbolRefExpr::VK_TLSDESC) {
570*0fca6ea1SDimitry Andric     const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(
571*0fca6ea1SDimitry Andric         MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)),
572*0fca6ea1SDimitry Andric         MCSymbolRefExpr::VK_TLSCALL, Ctx);
573*0fca6ea1SDimitry Andric     EmitAndCountInstruction(
574*0fca6ea1SDimitry Andric         MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)
575*0fca6ea1SDimitry Andric             .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX)
576*0fca6ea1SDimitry Andric             .addReg(Is64Bits ? X86::RIP : X86::EBX)
577*0fca6ea1SDimitry Andric             .addImm(1)
578*0fca6ea1SDimitry Andric             .addReg(0)
579*0fca6ea1SDimitry Andric             .addExpr(Sym)
580*0fca6ea1SDimitry Andric             .addReg(0));
581*0fca6ea1SDimitry Andric     EmitAndCountInstruction(
582*0fca6ea1SDimitry Andric         MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m)
583*0fca6ea1SDimitry Andric             .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX)
584*0fca6ea1SDimitry Andric             .addImm(1)
585*0fca6ea1SDimitry Andric             .addReg(0)
586*0fca6ea1SDimitry Andric             .addExpr(Expr)
587*0fca6ea1SDimitry Andric             .addReg(0));
588*0fca6ea1SDimitry Andric   } else if (Is64Bits) {
5890b57cec5SDimitry Andric     bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD;
590e8d8bef9SDimitry Andric     if (NeedsPadding && Is64BitsLP64)
5910b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
5920b57cec5SDimitry Andric     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
5930b57cec5SDimitry Andric                                 .addReg(X86::RDI)
5940b57cec5SDimitry Andric                                 .addReg(X86::RIP)
5950b57cec5SDimitry Andric                                 .addImm(1)
5960b57cec5SDimitry Andric                                 .addReg(0)
5970b57cec5SDimitry Andric                                 .addExpr(Sym)
5980b57cec5SDimitry Andric                                 .addReg(0));
5990b57cec5SDimitry Andric     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr");
6000b57cec5SDimitry Andric     if (NeedsPadding) {
6010b57cec5SDimitry Andric       if (!UseGot)
6020b57cec5SDimitry Andric         EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
6030b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
6040b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));
6050b57cec5SDimitry Andric     }
6060b57cec5SDimitry Andric     if (UseGot) {
6070b57cec5SDimitry Andric       const MCExpr *Expr = MCSymbolRefExpr::create(
6080b57cec5SDimitry Andric           TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
6090b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
6100b57cec5SDimitry Andric                                   .addReg(X86::RIP)
6110b57cec5SDimitry Andric                                   .addImm(1)
6120b57cec5SDimitry Andric                                   .addReg(0)
6130b57cec5SDimitry Andric                                   .addExpr(Expr)
6140b57cec5SDimitry Andric                                   .addReg(0));
6150b57cec5SDimitry Andric     } else {
6160b57cec5SDimitry Andric       EmitAndCountInstruction(
6170b57cec5SDimitry Andric           MCInstBuilder(X86::CALL64pcrel32)
6180b57cec5SDimitry Andric               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
6190b57cec5SDimitry Andric                                                MCSymbolRefExpr::VK_PLT, Ctx)));
6200b57cec5SDimitry Andric     }
6210b57cec5SDimitry Andric   } else {
6220b57cec5SDimitry Andric     if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) {
6230b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
6240b57cec5SDimitry Andric                                   .addReg(X86::EAX)
6250b57cec5SDimitry Andric                                   .addReg(0)
6260b57cec5SDimitry Andric                                   .addImm(1)
6270b57cec5SDimitry Andric                                   .addReg(X86::EBX)
6280b57cec5SDimitry Andric                                   .addExpr(Sym)
6290b57cec5SDimitry Andric                                   .addReg(0));
6300b57cec5SDimitry Andric     } else {
6310b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
6320b57cec5SDimitry Andric                                   .addReg(X86::EAX)
6330b57cec5SDimitry Andric                                   .addReg(X86::EBX)
6340b57cec5SDimitry Andric                                   .addImm(1)
6350b57cec5SDimitry Andric                                   .addReg(0)
6360b57cec5SDimitry Andric                                   .addExpr(Sym)
6370b57cec5SDimitry Andric                                   .addReg(0));
6380b57cec5SDimitry Andric     }
6390b57cec5SDimitry Andric 
6400b57cec5SDimitry Andric     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
6410b57cec5SDimitry Andric     if (UseGot) {
6420b57cec5SDimitry Andric       const MCExpr *Expr =
6430b57cec5SDimitry Andric           MCSymbolRefExpr::create(TlsGetAddr, MCSymbolRefExpr::VK_GOT, Ctx);
6440b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
6450b57cec5SDimitry Andric                                   .addReg(X86::EBX)
6460b57cec5SDimitry Andric                                   .addImm(1)
6470b57cec5SDimitry Andric                                   .addReg(0)
6480b57cec5SDimitry Andric                                   .addExpr(Expr)
6490b57cec5SDimitry Andric                                   .addReg(0));
6500b57cec5SDimitry Andric     } else {
6510b57cec5SDimitry Andric       EmitAndCountInstruction(
6520b57cec5SDimitry Andric           MCInstBuilder(X86::CALLpcrel32)
6530b57cec5SDimitry Andric               .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
6540b57cec5SDimitry Andric                                                MCSymbolRefExpr::VK_PLT, Ctx)));
6550b57cec5SDimitry Andric     }
6560b57cec5SDimitry Andric   }
6570b57cec5SDimitry Andric }
6580b57cec5SDimitry Andric 
6590b57cec5SDimitry Andric /// Emit the largest nop instruction smaller than or equal to \p NumBytes
6600b57cec5SDimitry Andric /// bytes.  Return the size of nop emitted.
6615ffd83dbSDimitry Andric static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
6625ffd83dbSDimitry Andric                         const X86Subtarget *Subtarget) {
663e8d8bef9SDimitry Andric   // Determine the longest nop which can be efficiently decoded for the given
664e8d8bef9SDimitry Andric   // target cpu.  15-bytes is the longest single NOP instruction, but some
665e8d8bef9SDimitry Andric   // platforms can't decode the longest forms efficiently.
666e8d8bef9SDimitry Andric   unsigned MaxNopLength = 1;
667e8d8bef9SDimitry Andric   if (Subtarget->is64Bit()) {
668e8d8bef9SDimitry Andric     // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
669e8d8bef9SDimitry Andric     // IndexReg/BaseReg below need to be updated.
670349cc55cSDimitry Andric     if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))
671e8d8bef9SDimitry Andric       MaxNopLength = 7;
672349cc55cSDimitry Andric     else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))
673e8d8bef9SDimitry Andric       MaxNopLength = 15;
674349cc55cSDimitry Andric     else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))
675e8d8bef9SDimitry Andric       MaxNopLength = 11;
676e8d8bef9SDimitry Andric     else
677e8d8bef9SDimitry Andric       MaxNopLength = 10;
678e8d8bef9SDimitry Andric   } if (Subtarget->is32Bit())
679e8d8bef9SDimitry Andric     MaxNopLength = 2;
680e8d8bef9SDimitry Andric 
681480093f4SDimitry Andric   // Cap a single nop emission at the profitable value for the target
682e8d8bef9SDimitry Andric   NumBytes = std::min(NumBytes, MaxNopLength);
6830b57cec5SDimitry Andric 
6840b57cec5SDimitry Andric   unsigned NopSize;
6850b57cec5SDimitry Andric   unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
6860b57cec5SDimitry Andric   IndexReg = Displacement = SegmentReg = 0;
6870b57cec5SDimitry Andric   BaseReg = X86::RAX;
6880b57cec5SDimitry Andric   ScaleVal = 1;
6890b57cec5SDimitry Andric   switch (NumBytes) {
6900b57cec5SDimitry Andric   case 0:
6910b57cec5SDimitry Andric     llvm_unreachable("Zero nops?");
6920b57cec5SDimitry Andric     break;
6930b57cec5SDimitry Andric   case 1:
6940b57cec5SDimitry Andric     NopSize = 1;
6950b57cec5SDimitry Andric     Opc = X86::NOOP;
6960b57cec5SDimitry Andric     break;
6970b57cec5SDimitry Andric   case 2:
6980b57cec5SDimitry Andric     NopSize = 2;
6990b57cec5SDimitry Andric     Opc = X86::XCHG16ar;
7000b57cec5SDimitry Andric     break;
7010b57cec5SDimitry Andric   case 3:
7020b57cec5SDimitry Andric     NopSize = 3;
7030b57cec5SDimitry Andric     Opc = X86::NOOPL;
7040b57cec5SDimitry Andric     break;
7050b57cec5SDimitry Andric   case 4:
7060b57cec5SDimitry Andric     NopSize = 4;
7070b57cec5SDimitry Andric     Opc = X86::NOOPL;
7080b57cec5SDimitry Andric     Displacement = 8;
7090b57cec5SDimitry Andric     break;
7100b57cec5SDimitry Andric   case 5:
7110b57cec5SDimitry Andric     NopSize = 5;
7120b57cec5SDimitry Andric     Opc = X86::NOOPL;
7130b57cec5SDimitry Andric     Displacement = 8;
7140b57cec5SDimitry Andric     IndexReg = X86::RAX;
7150b57cec5SDimitry Andric     break;
7160b57cec5SDimitry Andric   case 6:
7170b57cec5SDimitry Andric     NopSize = 6;
7180b57cec5SDimitry Andric     Opc = X86::NOOPW;
7190b57cec5SDimitry Andric     Displacement = 8;
7200b57cec5SDimitry Andric     IndexReg = X86::RAX;
7210b57cec5SDimitry Andric     break;
7220b57cec5SDimitry Andric   case 7:
7230b57cec5SDimitry Andric     NopSize = 7;
7240b57cec5SDimitry Andric     Opc = X86::NOOPL;
7250b57cec5SDimitry Andric     Displacement = 512;
7260b57cec5SDimitry Andric     break;
7270b57cec5SDimitry Andric   case 8:
7280b57cec5SDimitry Andric     NopSize = 8;
7290b57cec5SDimitry Andric     Opc = X86::NOOPL;
7300b57cec5SDimitry Andric     Displacement = 512;
7310b57cec5SDimitry Andric     IndexReg = X86::RAX;
7320b57cec5SDimitry Andric     break;
7330b57cec5SDimitry Andric   case 9:
7340b57cec5SDimitry Andric     NopSize = 9;
7350b57cec5SDimitry Andric     Opc = X86::NOOPW;
7360b57cec5SDimitry Andric     Displacement = 512;
7370b57cec5SDimitry Andric     IndexReg = X86::RAX;
7380b57cec5SDimitry Andric     break;
7390b57cec5SDimitry Andric   default:
7400b57cec5SDimitry Andric     NopSize = 10;
7410b57cec5SDimitry Andric     Opc = X86::NOOPW;
7420b57cec5SDimitry Andric     Displacement = 512;
7430b57cec5SDimitry Andric     IndexReg = X86::RAX;
7440b57cec5SDimitry Andric     SegmentReg = X86::CS;
7450b57cec5SDimitry Andric     break;
7460b57cec5SDimitry Andric   }
7470b57cec5SDimitry Andric 
7480b57cec5SDimitry Andric   unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);
7490b57cec5SDimitry Andric   NopSize += NumPrefixes;
7500b57cec5SDimitry Andric   for (unsigned i = 0; i != NumPrefixes; ++i)
7515ffd83dbSDimitry Andric     OS.emitBytes("\x66");
7520b57cec5SDimitry Andric 
7530b57cec5SDimitry Andric   switch (Opc) {
7540b57cec5SDimitry Andric   default: llvm_unreachable("Unexpected opcode");
7550b57cec5SDimitry Andric   case X86::NOOP:
7565ffd83dbSDimitry Andric     OS.emitInstruction(MCInstBuilder(Opc), *Subtarget);
7570b57cec5SDimitry Andric     break;
7580b57cec5SDimitry Andric   case X86::XCHG16ar:
7595ffd83dbSDimitry Andric     OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX),
7605ffd83dbSDimitry Andric                        *Subtarget);
7610b57cec5SDimitry Andric     break;
7620b57cec5SDimitry Andric   case X86::NOOPL:
7630b57cec5SDimitry Andric   case X86::NOOPW:
7645ffd83dbSDimitry Andric     OS.emitInstruction(MCInstBuilder(Opc)
7650b57cec5SDimitry Andric                            .addReg(BaseReg)
7660b57cec5SDimitry Andric                            .addImm(ScaleVal)
7670b57cec5SDimitry Andric                            .addReg(IndexReg)
7680b57cec5SDimitry Andric                            .addImm(Displacement)
7690b57cec5SDimitry Andric                            .addReg(SegmentReg),
7705ffd83dbSDimitry Andric                        *Subtarget);
7710b57cec5SDimitry Andric     break;
7720b57cec5SDimitry Andric   }
7730b57cec5SDimitry Andric   assert(NopSize <= NumBytes && "We overemitted?");
7740b57cec5SDimitry Andric   return NopSize;
7750b57cec5SDimitry Andric }
7760b57cec5SDimitry Andric 
7770b57cec5SDimitry Andric /// Emit the optimal amount of multi-byte nops on X86.
7785ffd83dbSDimitry Andric static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
7795ffd83dbSDimitry Andric                         const X86Subtarget *Subtarget) {
7800b57cec5SDimitry Andric   unsigned NopsToEmit = NumBytes;
7810b57cec5SDimitry Andric   (void)NopsToEmit;
7820b57cec5SDimitry Andric   while (NumBytes) {
7835ffd83dbSDimitry Andric     NumBytes -= emitNop(OS, NumBytes, Subtarget);
7840b57cec5SDimitry Andric     assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
7850b57cec5SDimitry Andric   }
7860b57cec5SDimitry Andric }
7870b57cec5SDimitry Andric 
7880b57cec5SDimitry Andric void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
7890b57cec5SDimitry Andric                                     X86MCInstLower &MCIL) {
7900b57cec5SDimitry Andric   assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
7910b57cec5SDimitry Andric 
792480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
793480093f4SDimitry Andric 
7940b57cec5SDimitry Andric   StatepointOpers SOpers(&MI);
7950b57cec5SDimitry Andric   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
7965ffd83dbSDimitry Andric     emitX86Nops(*OutStreamer, PatchBytes, Subtarget);
7970b57cec5SDimitry Andric   } else {
7980b57cec5SDimitry Andric     // Lower call target and choose correct opcode
7990b57cec5SDimitry Andric     const MachineOperand &CallTarget = SOpers.getCallTarget();
8000b57cec5SDimitry Andric     MCOperand CallTargetMCOp;
8010b57cec5SDimitry Andric     unsigned CallOpcode;
8020b57cec5SDimitry Andric     switch (CallTarget.getType()) {
8030b57cec5SDimitry Andric     case MachineOperand::MO_GlobalAddress:
8040b57cec5SDimitry Andric     case MachineOperand::MO_ExternalSymbol:
8050b57cec5SDimitry Andric       CallTargetMCOp = MCIL.LowerSymbolOperand(
8060b57cec5SDimitry Andric           CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
8070b57cec5SDimitry Andric       CallOpcode = X86::CALL64pcrel32;
8080b57cec5SDimitry Andric       // Currently, we only support relative addressing with statepoints.
8090b57cec5SDimitry Andric       // Otherwise, we'll need a scratch register to hold the target
8100b57cec5SDimitry Andric       // address.  You'll fail asserts during load & relocation if this
8110b57cec5SDimitry Andric       // symbol is to far away. (TODO: support non-relative addressing)
8120b57cec5SDimitry Andric       break;
8130b57cec5SDimitry Andric     case MachineOperand::MO_Immediate:
8140b57cec5SDimitry Andric       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
8150b57cec5SDimitry Andric       CallOpcode = X86::CALL64pcrel32;
8160b57cec5SDimitry Andric       // Currently, we only support relative addressing with statepoints.
8170b57cec5SDimitry Andric       // Otherwise, we'll need a scratch register to hold the target
8180b57cec5SDimitry Andric       // immediate.  You'll fail asserts during load & relocation if this
8190b57cec5SDimitry Andric       // address is to far away. (TODO: support non-relative addressing)
8200b57cec5SDimitry Andric       break;
8210b57cec5SDimitry Andric     case MachineOperand::MO_Register:
8220b57cec5SDimitry Andric       // FIXME: Add retpoline support and remove this.
8230946e70aSDimitry Andric       if (Subtarget->useIndirectThunkCalls())
8240946e70aSDimitry Andric         report_fatal_error("Lowering register statepoints with thunks not "
8250b57cec5SDimitry Andric                            "yet implemented.");
8260b57cec5SDimitry Andric       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
8270b57cec5SDimitry Andric       CallOpcode = X86::CALL64r;
8280b57cec5SDimitry Andric       break;
8290b57cec5SDimitry Andric     default:
8300b57cec5SDimitry Andric       llvm_unreachable("Unsupported operand type in statepoint call target");
8310b57cec5SDimitry Andric       break;
8320b57cec5SDimitry Andric     }
8330b57cec5SDimitry Andric 
8340b57cec5SDimitry Andric     // Emit call
8350b57cec5SDimitry Andric     MCInst CallInst;
8360b57cec5SDimitry Andric     CallInst.setOpcode(CallOpcode);
8370b57cec5SDimitry Andric     CallInst.addOperand(CallTargetMCOp);
8385ffd83dbSDimitry Andric     OutStreamer->emitInstruction(CallInst, getSubtargetInfo());
8390b57cec5SDimitry Andric   }
8400b57cec5SDimitry Andric 
8410b57cec5SDimitry Andric   // Record our statepoint node in the same section used by STACKMAP
8420b57cec5SDimitry Andric   // and PATCHPOINT
843480093f4SDimitry Andric   auto &Ctx = OutStreamer->getContext();
844480093f4SDimitry Andric   MCSymbol *MILabel = Ctx.createTempSymbol();
8455ffd83dbSDimitry Andric   OutStreamer->emitLabel(MILabel);
846480093f4SDimitry Andric   SM.recordStatepoint(*MILabel, MI);
8470b57cec5SDimitry Andric }
8480b57cec5SDimitry Andric 
8490b57cec5SDimitry Andric void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
8500b57cec5SDimitry Andric                                      X86MCInstLower &MCIL) {
8510b57cec5SDimitry Andric   // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
8520b57cec5SDimitry Andric   //                  <opcode>, <operands>
8530b57cec5SDimitry Andric 
854480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
855480093f4SDimitry Andric 
8568bcb0991SDimitry Andric   Register DefRegister = FaultingMI.getOperand(0).getReg();
8570b57cec5SDimitry Andric   FaultMaps::FaultKind FK =
8580b57cec5SDimitry Andric       static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
8590b57cec5SDimitry Andric   MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
8600b57cec5SDimitry Andric   unsigned Opcode = FaultingMI.getOperand(3).getImm();
8610b57cec5SDimitry Andric   unsigned OperandsBeginIdx = 4;
8620b57cec5SDimitry Andric 
863480093f4SDimitry Andric   auto &Ctx = OutStreamer->getContext();
864480093f4SDimitry Andric   MCSymbol *FaultingLabel = Ctx.createTempSymbol();
8655ffd83dbSDimitry Andric   OutStreamer->emitLabel(FaultingLabel);
866480093f4SDimitry Andric 
8670b57cec5SDimitry Andric   assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
868480093f4SDimitry Andric   FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
8690b57cec5SDimitry Andric 
8700b57cec5SDimitry Andric   MCInst MI;
8710b57cec5SDimitry Andric   MI.setOpcode(Opcode);
8720b57cec5SDimitry Andric 
8730b57cec5SDimitry Andric   if (DefRegister != X86::NoRegister)
8740b57cec5SDimitry Andric     MI.addOperand(MCOperand::createReg(DefRegister));
8750b57cec5SDimitry Andric 
876bdd1243dSDimitry Andric   for (const MachineOperand &MO :
877bdd1243dSDimitry Andric        llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx))
878*0fca6ea1SDimitry Andric     if (auto Op = MCIL.LowerMachineOperand(&FaultingMI, MO); Op.isValid())
879*0fca6ea1SDimitry Andric       MI.addOperand(Op);
8800b57cec5SDimitry Andric 
8810b57cec5SDimitry Andric   OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
8825ffd83dbSDimitry Andric   OutStreamer->emitInstruction(MI, getSubtargetInfo());
8830b57cec5SDimitry Andric }
8840b57cec5SDimitry Andric 
8850b57cec5SDimitry Andric void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
8860b57cec5SDimitry Andric                                      X86MCInstLower &MCIL) {
8870b57cec5SDimitry Andric   bool Is64Bits = Subtarget->is64Bit();
8880b57cec5SDimitry Andric   MCContext &Ctx = OutStreamer->getContext();
8890b57cec5SDimitry Andric   MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
8900b57cec5SDimitry Andric   const MCSymbolRefExpr *Op =
8910b57cec5SDimitry Andric       MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_None, Ctx);
8920b57cec5SDimitry Andric 
8930b57cec5SDimitry Andric   EmitAndCountInstruction(
8940b57cec5SDimitry Andric       MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
8950b57cec5SDimitry Andric           .addExpr(Op));
8960b57cec5SDimitry Andric }
8970b57cec5SDimitry Andric 
898bdd1243dSDimitry Andric void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
899bdd1243dSDimitry Andric   assert(std::next(MI.getIterator())->isCall() &&
900bdd1243dSDimitry Andric          "KCFI_CHECK not followed by a call instruction");
901bdd1243dSDimitry Andric 
902bdd1243dSDimitry Andric   // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()
903bdd1243dSDimitry Andric   // returns a 1-byte X86::NOOP, which means the offset is the same in
904bdd1243dSDimitry Andric   // bytes.  This assumes that patchable-function-prefix is the same for all
905bdd1243dSDimitry Andric   // functions.
906bdd1243dSDimitry Andric   const MachineFunction &MF = *MI.getMF();
907bdd1243dSDimitry Andric   int64_t PrefixNops = 0;
908bdd1243dSDimitry Andric   (void)MF.getFunction()
909bdd1243dSDimitry Andric       .getFnAttribute("patchable-function-prefix")
910bdd1243dSDimitry Andric       .getValueAsString()
911bdd1243dSDimitry Andric       .getAsInteger(10, PrefixNops);
912bdd1243dSDimitry Andric 
913bdd1243dSDimitry Andric   // KCFI allows indirect calls to any location that's preceded by a valid
914bdd1243dSDimitry Andric   // type identifier. To avoid encoding the full constant into an instruction,
915bdd1243dSDimitry Andric   // and thus emitting potential call target gadgets at each indirect call
916bdd1243dSDimitry Andric   // site, load a negated constant to a register and compare that to the
917bdd1243dSDimitry Andric   // expected value at the call target.
918bdd1243dSDimitry Andric   const Register AddrReg = MI.getOperand(0).getReg();
919bdd1243dSDimitry Andric   const uint32_t Type = MI.getOperand(1).getImm();
920bdd1243dSDimitry Andric   // The check is immediately before the call. If the call target is in R10,
921bdd1243dSDimitry Andric   // we can clobber R11 for the check instead.
922bdd1243dSDimitry Andric   unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;
923bdd1243dSDimitry Andric   EmitAndCountInstruction(
924bdd1243dSDimitry Andric       MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type)));
925bdd1243dSDimitry Andric   EmitAndCountInstruction(MCInstBuilder(X86::ADD32rm)
926bdd1243dSDimitry Andric                               .addReg(X86::NoRegister)
927bdd1243dSDimitry Andric                               .addReg(TempReg)
928bdd1243dSDimitry Andric                               .addReg(AddrReg)
929bdd1243dSDimitry Andric                               .addImm(1)
930bdd1243dSDimitry Andric                               .addReg(X86::NoRegister)
931bdd1243dSDimitry Andric                               .addImm(-(PrefixNops + 4))
932bdd1243dSDimitry Andric                               .addReg(X86::NoRegister));
933bdd1243dSDimitry Andric 
934bdd1243dSDimitry Andric   MCSymbol *Pass = OutContext.createTempSymbol();
935bdd1243dSDimitry Andric   EmitAndCountInstruction(
936bdd1243dSDimitry Andric       MCInstBuilder(X86::JCC_1)
937bdd1243dSDimitry Andric           .addExpr(MCSymbolRefExpr::create(Pass, OutContext))
938bdd1243dSDimitry Andric           .addImm(X86::COND_E));
939bdd1243dSDimitry Andric 
940bdd1243dSDimitry Andric   MCSymbol *Trap = OutContext.createTempSymbol();
941bdd1243dSDimitry Andric   OutStreamer->emitLabel(Trap);
942bdd1243dSDimitry Andric   EmitAndCountInstruction(MCInstBuilder(X86::TRAP));
943bdd1243dSDimitry Andric   emitKCFITrapEntry(MF, Trap);
944bdd1243dSDimitry Andric   OutStreamer->emitLabel(Pass);
945bdd1243dSDimitry Andric }
946bdd1243dSDimitry Andric 
947349cc55cSDimitry Andric void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
948349cc55cSDimitry Andric   // FIXME: Make this work on non-ELF.
949349cc55cSDimitry Andric   if (!TM.getTargetTriple().isOSBinFormatELF()) {
950349cc55cSDimitry Andric     report_fatal_error("llvm.asan.check.memaccess only supported on ELF");
951349cc55cSDimitry Andric     return;
952349cc55cSDimitry Andric   }
953349cc55cSDimitry Andric 
9540eae32dcSDimitry Andric   const auto &Reg = MI.getOperand(0).getReg();
955349cc55cSDimitry Andric   ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());
956349cc55cSDimitry Andric 
9570eae32dcSDimitry Andric   uint64_t ShadowBase;
9580eae32dcSDimitry Andric   int MappingScale;
9590eae32dcSDimitry Andric   bool OrShadowOffset;
9600eae32dcSDimitry Andric   getAddressSanitizerParams(Triple(TM.getTargetTriple()), 64,
9610eae32dcSDimitry Andric                             AccessInfo.CompileKernel, &ShadowBase,
9620eae32dcSDimitry Andric                             &MappingScale, &OrShadowOffset);
9630eae32dcSDimitry Andric 
96481ad6265SDimitry Andric   StringRef Name = AccessInfo.IsWrite ? "store" : "load";
96581ad6265SDimitry Andric   StringRef Op = OrShadowOffset ? "or" : "add";
96681ad6265SDimitry Andric   std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" +
96781ad6265SDimitry Andric                          Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" +
96881ad6265SDimitry Andric                          TM.getMCRegisterInfo()->getName(Reg.asMCReg()))
96981ad6265SDimitry Andric                             .str();
9700eae32dcSDimitry Andric   if (OrShadowOffset)
9710eae32dcSDimitry Andric     report_fatal_error(
9720eae32dcSDimitry Andric         "OrShadowOffset is not supported with optimized callbacks");
973349cc55cSDimitry Andric 
974349cc55cSDimitry Andric   EmitAndCountInstruction(
975349cc55cSDimitry Andric       MCInstBuilder(X86::CALL64pcrel32)
9760eae32dcSDimitry Andric           .addExpr(MCSymbolRefExpr::create(
9770eae32dcSDimitry Andric               OutContext.getOrCreateSymbol(SymName), OutContext)));
978349cc55cSDimitry Andric }
979349cc55cSDimitry Andric 
9800b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
9810b57cec5SDimitry Andric                                       X86MCInstLower &MCIL) {
9827a6dacacSDimitry Andric   // PATCHABLE_OP minsize
9830b57cec5SDimitry Andric 
984480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
985480093f4SDimitry Andric 
9867a6dacacSDimitry Andric   auto NextMI = std::find_if(std::next(MI.getIterator()),
9877a6dacacSDimitry Andric                              MI.getParent()->end().getInstrIterator(),
9887a6dacacSDimitry Andric                              [](auto &II) { return !II.isMetaInstruction(); });
9890b57cec5SDimitry Andric 
9900b57cec5SDimitry Andric   SmallString<256> Code;
9917a6dacacSDimitry Andric   unsigned MinSize = MI.getOperand(0).getImm();
9927a6dacacSDimitry Andric 
993dfa39133SDimitry Andric   if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) {
9947a6dacacSDimitry Andric     // Lower the next MachineInstr to find its byte size.
995dfa39133SDimitry Andric     // If the next instruction is inline assembly, we skip lowering it for now,
996dfa39133SDimitry Andric     // and assume we should always generate NOPs.
9977a6dacacSDimitry Andric     MCInst MCI;
9987a6dacacSDimitry Andric     MCIL.Lower(&*NextMI, MCI);
9997a6dacacSDimitry Andric 
10000b57cec5SDimitry Andric     SmallVector<MCFixup, 4> Fixups;
100106c3fb27SDimitry Andric     CodeEmitter->encodeInstruction(MCI, Code, Fixups, getSubtargetInfo());
1002bdd1243dSDimitry Andric   }
10030b57cec5SDimitry Andric 
10040b57cec5SDimitry Andric   if (Code.size() < MinSize) {
10055ffd83dbSDimitry Andric     if (MinSize == 2 && Subtarget->is32Bit() &&
10065ffd83dbSDimitry Andric         Subtarget->isTargetWindowsMSVC() &&
10075ffd83dbSDimitry Andric         (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
10085f757f3fSDimitry Andric       // For compatibility reasons, when targetting MSVC, it is important to
10095ffd83dbSDimitry Andric       // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
10105ffd83dbSDimitry Andric       // rely specifically on this pattern to be able to patch a function.
10115ffd83dbSDimitry Andric       // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
10125ffd83dbSDimitry Andric       OutStreamer->emitInstruction(
10135ffd83dbSDimitry Andric           MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI),
10145ffd83dbSDimitry Andric           *Subtarget);
10150b57cec5SDimitry Andric     } else {
10165ffd83dbSDimitry Andric       unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget);
10170b57cec5SDimitry Andric       assert(NopSize == MinSize && "Could not implement MinSize!");
10180b57cec5SDimitry Andric       (void)NopSize;
10190b57cec5SDimitry Andric     }
10200b57cec5SDimitry Andric   }
10210b57cec5SDimitry Andric }
10220b57cec5SDimitry Andric 
10230b57cec5SDimitry Andric // Lower a stackmap of the form:
10240b57cec5SDimitry Andric // <id>, <shadowBytes>, ...
10250b57cec5SDimitry Andric void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
10260b57cec5SDimitry Andric   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1027480093f4SDimitry Andric 
1028480093f4SDimitry Andric   auto &Ctx = OutStreamer->getContext();
1029480093f4SDimitry Andric   MCSymbol *MILabel = Ctx.createTempSymbol();
10305ffd83dbSDimitry Andric   OutStreamer->emitLabel(MILabel);
1031480093f4SDimitry Andric 
1032480093f4SDimitry Andric   SM.recordStackMap(*MILabel, MI);
10330b57cec5SDimitry Andric   unsigned NumShadowBytes = MI.getOperand(1).getImm();
10340b57cec5SDimitry Andric   SMShadowTracker.reset(NumShadowBytes);
10350b57cec5SDimitry Andric }
10360b57cec5SDimitry Andric 
10370b57cec5SDimitry Andric // Lower a patchpoint of the form:
10380b57cec5SDimitry Andric // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
10390b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
10400b57cec5SDimitry Andric                                     X86MCInstLower &MCIL) {
10410b57cec5SDimitry Andric   assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
10420b57cec5SDimitry Andric 
10430b57cec5SDimitry Andric   SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
10440b57cec5SDimitry Andric 
1045480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
1046480093f4SDimitry Andric 
1047480093f4SDimitry Andric   auto &Ctx = OutStreamer->getContext();
1048480093f4SDimitry Andric   MCSymbol *MILabel = Ctx.createTempSymbol();
10495ffd83dbSDimitry Andric   OutStreamer->emitLabel(MILabel);
1050480093f4SDimitry Andric   SM.recordPatchPoint(*MILabel, MI);
10510b57cec5SDimitry Andric 
10520b57cec5SDimitry Andric   PatchPointOpers opers(&MI);
10530b57cec5SDimitry Andric   unsigned ScratchIdx = opers.getNextScratchIdx();
10540b57cec5SDimitry Andric   unsigned EncodedBytes = 0;
10550b57cec5SDimitry Andric   const MachineOperand &CalleeMO = opers.getCallTarget();
10560b57cec5SDimitry Andric 
10570b57cec5SDimitry Andric   // Check for null target. If target is non-null (i.e. is non-zero or is
10580b57cec5SDimitry Andric   // symbolic) then emit a call.
10590b57cec5SDimitry Andric   if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
10600b57cec5SDimitry Andric     MCOperand CalleeMCOp;
10610b57cec5SDimitry Andric     switch (CalleeMO.getType()) {
10620b57cec5SDimitry Andric     default:
10630b57cec5SDimitry Andric       /// FIXME: Add a verifier check for bad callee types.
10640b57cec5SDimitry Andric       llvm_unreachable("Unrecognized callee operand type.");
10650b57cec5SDimitry Andric     case MachineOperand::MO_Immediate:
10660b57cec5SDimitry Andric       if (CalleeMO.getImm())
10670b57cec5SDimitry Andric         CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());
10680b57cec5SDimitry Andric       break;
10690b57cec5SDimitry Andric     case MachineOperand::MO_ExternalSymbol:
10700b57cec5SDimitry Andric     case MachineOperand::MO_GlobalAddress:
10710b57cec5SDimitry Andric       CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO,
10720b57cec5SDimitry Andric                                            MCIL.GetSymbolFromOperand(CalleeMO));
10730b57cec5SDimitry Andric       break;
10740b57cec5SDimitry Andric     }
10750b57cec5SDimitry Andric 
10760b57cec5SDimitry Andric     // Emit MOV to materialize the target address and the CALL to target.
10770b57cec5SDimitry Andric     // This is encoded with 12-13 bytes, depending on which register is used.
10788bcb0991SDimitry Andric     Register ScratchReg = MI.getOperand(ScratchIdx).getReg();
10790b57cec5SDimitry Andric     if (X86II::isX86_64ExtendedReg(ScratchReg))
10800b57cec5SDimitry Andric       EncodedBytes = 13;
10810b57cec5SDimitry Andric     else
10820b57cec5SDimitry Andric       EncodedBytes = 12;
10830b57cec5SDimitry Andric 
10840b57cec5SDimitry Andric     EmitAndCountInstruction(
10850b57cec5SDimitry Andric         MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
10860b57cec5SDimitry Andric     // FIXME: Add retpoline support and remove this.
10870946e70aSDimitry Andric     if (Subtarget->useIndirectThunkCalls())
10880b57cec5SDimitry Andric       report_fatal_error(
10890946e70aSDimitry Andric           "Lowering patchpoint with thunks not yet implemented.");
10900b57cec5SDimitry Andric     EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
10910b57cec5SDimitry Andric   }
10920b57cec5SDimitry Andric 
10930b57cec5SDimitry Andric   // Emit padding.
10940b57cec5SDimitry Andric   unsigned NumBytes = opers.getNumPatchBytes();
10950b57cec5SDimitry Andric   assert(NumBytes >= EncodedBytes &&
10960b57cec5SDimitry Andric          "Patchpoint can't request size less than the length of a call.");
10970b57cec5SDimitry Andric 
10985ffd83dbSDimitry Andric   emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);
10990b57cec5SDimitry Andric }
11000b57cec5SDimitry Andric 
11010b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
11020b57cec5SDimitry Andric                                               X86MCInstLower &MCIL) {
11030b57cec5SDimitry Andric   assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
11040b57cec5SDimitry Andric 
1105480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
1106480093f4SDimitry Andric 
11070b57cec5SDimitry Andric   // We want to emit the following pattern, which follows the x86 calling
11080b57cec5SDimitry Andric   // convention to prepare for the trampoline call to be patched in.
11090b57cec5SDimitry Andric   //
11100b57cec5SDimitry Andric   //   .p2align 1, ...
11110b57cec5SDimitry Andric   // .Lxray_event_sled_N:
11120b57cec5SDimitry Andric   //   jmp +N                        // jump across the instrumentation sled
11130b57cec5SDimitry Andric   //   ...                           // set up arguments in register
11140b57cec5SDimitry Andric   //   callq __xray_CustomEvent@plt  // force dependency to symbol
11150b57cec5SDimitry Andric   //   ...
11160b57cec5SDimitry Andric   //   <jump here>
11170b57cec5SDimitry Andric   //
11180b57cec5SDimitry Andric   // After patching, it would look something like:
11190b57cec5SDimitry Andric   //
11200b57cec5SDimitry Andric   //   nopw (2-byte nop)
11210b57cec5SDimitry Andric   //   ...
11220b57cec5SDimitry Andric   //   callq __xrayCustomEvent  // already lowered
11230b57cec5SDimitry Andric   //   ...
11240b57cec5SDimitry Andric   //
11250b57cec5SDimitry Andric   // ---
11260b57cec5SDimitry Andric   // First we emit the label and the jump.
11270b57cec5SDimitry Andric   auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true);
11280b57cec5SDimitry Andric   OutStreamer->AddComment("# XRay Custom Event Log");
1129bdd1243dSDimitry Andric   OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
11305ffd83dbSDimitry Andric   OutStreamer->emitLabel(CurSled);
11310b57cec5SDimitry Andric 
11320b57cec5SDimitry Andric   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
11330b57cec5SDimitry Andric   // an operand (computed as an offset from the jmp instruction).
11340b57cec5SDimitry Andric   // FIXME: Find another less hacky way do force the relative jump.
11355ffd83dbSDimitry Andric   OutStreamer->emitBinaryData("\xeb\x0f");
11360b57cec5SDimitry Andric 
11370b57cec5SDimitry Andric   // The default C calling convention will place two arguments into %rcx and
11380b57cec5SDimitry Andric   // %rdx -- so we only work with those.
1139480093f4SDimitry Andric   const Register DestRegs[] = {X86::RDI, X86::RSI};
11400b57cec5SDimitry Andric   bool UsedMask[] = {false, false};
11410b57cec5SDimitry Andric   // Filled out in loop.
1142480093f4SDimitry Andric   Register SrcRegs[] = {0, 0};
11430b57cec5SDimitry Andric 
11440b57cec5SDimitry Andric   // Then we put the operands in the %rdi and %rsi registers. We spill the
11450b57cec5SDimitry Andric   // values in the register before we clobber them, and mark them as used in
11460b57cec5SDimitry Andric   // UsedMask. In case the arguments are already in the correct register, we use
11470b57cec5SDimitry Andric   // emit nops appropriately sized to keep the sled the same size in every
11480b57cec5SDimitry Andric   // situation.
11490b57cec5SDimitry Andric   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1150*0fca6ea1SDimitry Andric     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I));
1151*0fca6ea1SDimitry Andric         Op.isValid()) {
1152*0fca6ea1SDimitry Andric       assert(Op.isReg() && "Only support arguments in registers");
1153*0fca6ea1SDimitry Andric       SrcRegs[I] = getX86SubSuperRegister(Op.getReg(), 64);
115406c3fb27SDimitry Andric       assert(SrcRegs[I].isValid() && "Invalid operand");
11550b57cec5SDimitry Andric       if (SrcRegs[I] != DestRegs[I]) {
11560b57cec5SDimitry Andric         UsedMask[I] = true;
11570b57cec5SDimitry Andric         EmitAndCountInstruction(
11580b57cec5SDimitry Andric             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
11590b57cec5SDimitry Andric       } else {
11605ffd83dbSDimitry Andric         emitX86Nops(*OutStreamer, 4, Subtarget);
11610b57cec5SDimitry Andric       }
11620b57cec5SDimitry Andric     }
11630b57cec5SDimitry Andric 
11640b57cec5SDimitry Andric   // Now that the register values are stashed, mov arguments into place.
1165480093f4SDimitry Andric   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1166480093f4SDimitry Andric   // earlier DestReg. We will have already overwritten over the register before
1167480093f4SDimitry Andric   // we can copy from it.
11680b57cec5SDimitry Andric   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
11690b57cec5SDimitry Andric     if (SrcRegs[I] != DestRegs[I])
11700b57cec5SDimitry Andric       EmitAndCountInstruction(
11710b57cec5SDimitry Andric           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
11720b57cec5SDimitry Andric 
11730b57cec5SDimitry Andric   // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
11740b57cec5SDimitry Andric   // name of the trampoline to be implemented by the XRay runtime.
11750b57cec5SDimitry Andric   auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent");
11760b57cec5SDimitry Andric   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
11770b57cec5SDimitry Andric   if (isPositionIndependent())
11780b57cec5SDimitry Andric     TOp.setTargetFlags(X86II::MO_PLT);
11790b57cec5SDimitry Andric 
11800b57cec5SDimitry Andric   // Emit the call instruction.
11810b57cec5SDimitry Andric   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
11820b57cec5SDimitry Andric                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
11830b57cec5SDimitry Andric 
11840b57cec5SDimitry Andric   // Restore caller-saved and used registers.
11850b57cec5SDimitry Andric   for (unsigned I = sizeof UsedMask; I-- > 0;)
11860b57cec5SDimitry Andric     if (UsedMask[I])
11870b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
11880b57cec5SDimitry Andric     else
11895ffd83dbSDimitry Andric       emitX86Nops(*OutStreamer, 1, Subtarget);
11900b57cec5SDimitry Andric 
11910b57cec5SDimitry Andric   OutStreamer->AddComment("xray custom event end.");
11920b57cec5SDimitry Andric 
11935ffd83dbSDimitry Andric   // Record the sled version. Version 0 of this sled was spelled differently, so
11945ffd83dbSDimitry Andric   // we let the runtime handle the different offsets we're using. Version 2
11955ffd83dbSDimitry Andric   // changed the absolute address to a PC-relative address.
11965ffd83dbSDimitry Andric   recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
11970b57cec5SDimitry Andric }
11980b57cec5SDimitry Andric 
11990b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
12000b57cec5SDimitry Andric                                                     X86MCInstLower &MCIL) {
12010b57cec5SDimitry Andric   assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
12020b57cec5SDimitry Andric 
1203480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
1204480093f4SDimitry Andric 
12050b57cec5SDimitry Andric   // We want to emit the following pattern, which follows the x86 calling
12060b57cec5SDimitry Andric   // convention to prepare for the trampoline call to be patched in.
12070b57cec5SDimitry Andric   //
12080b57cec5SDimitry Andric   //   .p2align 1, ...
12090b57cec5SDimitry Andric   // .Lxray_event_sled_N:
12100b57cec5SDimitry Andric   //   jmp +N                        // jump across the instrumentation sled
12110b57cec5SDimitry Andric   //   ...                           // set up arguments in register
12120b57cec5SDimitry Andric   //   callq __xray_TypedEvent@plt  // force dependency to symbol
12130b57cec5SDimitry Andric   //   ...
12140b57cec5SDimitry Andric   //   <jump here>
12150b57cec5SDimitry Andric   //
12160b57cec5SDimitry Andric   // After patching, it would look something like:
12170b57cec5SDimitry Andric   //
12180b57cec5SDimitry Andric   //   nopw (2-byte nop)
12190b57cec5SDimitry Andric   //   ...
12200b57cec5SDimitry Andric   //   callq __xrayTypedEvent  // already lowered
12210b57cec5SDimitry Andric   //   ...
12220b57cec5SDimitry Andric   //
12230b57cec5SDimitry Andric   // ---
12240b57cec5SDimitry Andric   // First we emit the label and the jump.
12250b57cec5SDimitry Andric   auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true);
12260b57cec5SDimitry Andric   OutStreamer->AddComment("# XRay Typed Event Log");
1227bdd1243dSDimitry Andric   OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
12285ffd83dbSDimitry Andric   OutStreamer->emitLabel(CurSled);
12290b57cec5SDimitry Andric 
12300b57cec5SDimitry Andric   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
12310b57cec5SDimitry Andric   // an operand (computed as an offset from the jmp instruction).
12320b57cec5SDimitry Andric   // FIXME: Find another less hacky way do force the relative jump.
12335ffd83dbSDimitry Andric   OutStreamer->emitBinaryData("\xeb\x14");
12340b57cec5SDimitry Andric 
12350b57cec5SDimitry Andric   // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
12360b57cec5SDimitry Andric   // so we'll work with those. Or we may be called via SystemV, in which case
12370b57cec5SDimitry Andric   // we don't have to do any translation.
1238480093f4SDimitry Andric   const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
12390b57cec5SDimitry Andric   bool UsedMask[] = {false, false, false};
12400b57cec5SDimitry Andric 
12410b57cec5SDimitry Andric   // Will fill out src regs in the loop.
1242480093f4SDimitry Andric   Register SrcRegs[] = {0, 0, 0};
12430b57cec5SDimitry Andric 
12440b57cec5SDimitry Andric   // Then we put the operands in the SystemV registers. We spill the values in
12450b57cec5SDimitry Andric   // the registers before we clobber them, and mark them as used in UsedMask.
12460b57cec5SDimitry Andric   // In case the arguments are already in the correct register, we emit nops
12470b57cec5SDimitry Andric   // appropriately sized to keep the sled the same size in every situation.
12480b57cec5SDimitry Andric   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1249*0fca6ea1SDimitry Andric     if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I));
1250*0fca6ea1SDimitry Andric         Op.isValid()) {
12510b57cec5SDimitry Andric       // TODO: Is register only support adequate?
1252*0fca6ea1SDimitry Andric       assert(Op.isReg() && "Only supports arguments in registers");
1253*0fca6ea1SDimitry Andric       SrcRegs[I] = getX86SubSuperRegister(Op.getReg(), 64);
125406c3fb27SDimitry Andric       assert(SrcRegs[I].isValid() && "Invalid operand");
12550b57cec5SDimitry Andric       if (SrcRegs[I] != DestRegs[I]) {
12560b57cec5SDimitry Andric         UsedMask[I] = true;
12570b57cec5SDimitry Andric         EmitAndCountInstruction(
12580b57cec5SDimitry Andric             MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
12590b57cec5SDimitry Andric       } else {
12605ffd83dbSDimitry Andric         emitX86Nops(*OutStreamer, 4, Subtarget);
12610b57cec5SDimitry Andric       }
12620b57cec5SDimitry Andric     }
12630b57cec5SDimitry Andric 
12640b57cec5SDimitry Andric   // In the above loop we only stash all of the destination registers or emit
12650b57cec5SDimitry Andric   // nops if the arguments are already in the right place. Doing the actually
12660b57cec5SDimitry Andric   // moving is postponed until after all the registers are stashed so nothing
12670b57cec5SDimitry Andric   // is clobbers. We've already added nops to account for the size of mov and
12680b57cec5SDimitry Andric   // push if the register is in the right place, so we only have to worry about
12690b57cec5SDimitry Andric   // emitting movs.
1270480093f4SDimitry Andric   // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1271480093f4SDimitry Andric   // earlier DestReg. We will have already overwritten over the register before
1272480093f4SDimitry Andric   // we can copy from it.
12730b57cec5SDimitry Andric   for (unsigned I = 0; I < MI.getNumOperands(); ++I)
12740b57cec5SDimitry Andric     if (UsedMask[I])
12750b57cec5SDimitry Andric       EmitAndCountInstruction(
12760b57cec5SDimitry Andric           MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
12770b57cec5SDimitry Andric 
12780b57cec5SDimitry Andric   // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
12790b57cec5SDimitry Andric   // name of the trampoline to be implemented by the XRay runtime.
12800b57cec5SDimitry Andric   auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent");
12810b57cec5SDimitry Andric   MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);
12820b57cec5SDimitry Andric   if (isPositionIndependent())
12830b57cec5SDimitry Andric     TOp.setTargetFlags(X86II::MO_PLT);
12840b57cec5SDimitry Andric 
12850b57cec5SDimitry Andric   // Emit the call instruction.
12860b57cec5SDimitry Andric   EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
12870b57cec5SDimitry Andric                               .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
12880b57cec5SDimitry Andric 
12890b57cec5SDimitry Andric   // Restore caller-saved and used registers.
12900b57cec5SDimitry Andric   for (unsigned I = sizeof UsedMask; I-- > 0;)
12910b57cec5SDimitry Andric     if (UsedMask[I])
12920b57cec5SDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
12930b57cec5SDimitry Andric     else
12945ffd83dbSDimitry Andric       emitX86Nops(*OutStreamer, 1, Subtarget);
12950b57cec5SDimitry Andric 
12960b57cec5SDimitry Andric   OutStreamer->AddComment("xray typed event end.");
12970b57cec5SDimitry Andric 
12980b57cec5SDimitry Andric   // Record the sled version.
12995ffd83dbSDimitry Andric   recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
13000b57cec5SDimitry Andric }
13010b57cec5SDimitry Andric 
13020b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
13030b57cec5SDimitry Andric                                                   X86MCInstLower &MCIL) {
1304480093f4SDimitry Andric 
1305480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
1306480093f4SDimitry Andric 
1307480093f4SDimitry Andric   const Function &F = MF->getFunction();
1308480093f4SDimitry Andric   if (F.hasFnAttribute("patchable-function-entry")) {
1309480093f4SDimitry Andric     unsigned Num;
1310480093f4SDimitry Andric     if (F.getFnAttribute("patchable-function-entry")
1311480093f4SDimitry Andric             .getValueAsString()
1312480093f4SDimitry Andric             .getAsInteger(10, Num))
1313480093f4SDimitry Andric       return;
13145ffd83dbSDimitry Andric     emitX86Nops(*OutStreamer, Num, Subtarget);
1315480093f4SDimitry Andric     return;
1316480093f4SDimitry Andric   }
13170b57cec5SDimitry Andric   // We want to emit the following pattern:
13180b57cec5SDimitry Andric   //
13190b57cec5SDimitry Andric   //   .p2align 1, ...
13200b57cec5SDimitry Andric   // .Lxray_sled_N:
13210b57cec5SDimitry Andric   //   jmp .tmpN
13220b57cec5SDimitry Andric   //   # 9 bytes worth of noops
13230b57cec5SDimitry Andric   //
13240b57cec5SDimitry Andric   // We need the 9 bytes because at runtime, we'd be patching over the full 11
13250b57cec5SDimitry Andric   // bytes with the following pattern:
13260b57cec5SDimitry Andric   //
13270b57cec5SDimitry Andric   //   mov %r10, <function id, 32-bit>   // 6 bytes
13280b57cec5SDimitry Andric   //   call <relative offset, 32-bits>   // 5 bytes
13290b57cec5SDimitry Andric   //
13300b57cec5SDimitry Andric   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1331bdd1243dSDimitry Andric   OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
13325ffd83dbSDimitry Andric   OutStreamer->emitLabel(CurSled);
13330b57cec5SDimitry Andric 
13340b57cec5SDimitry Andric   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
13350b57cec5SDimitry Andric   // an operand (computed as an offset from the jmp instruction).
13360b57cec5SDimitry Andric   // FIXME: Find another less hacky way do force the relative jump.
13375ffd83dbSDimitry Andric   OutStreamer->emitBytes("\xeb\x09");
13385ffd83dbSDimitry Andric   emitX86Nops(*OutStreamer, 9, Subtarget);
13395ffd83dbSDimitry Andric   recordSled(CurSled, MI, SledKind::FUNCTION_ENTER, 2);
13400b57cec5SDimitry Andric }
13410b57cec5SDimitry Andric 
13420b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
13430b57cec5SDimitry Andric                                        X86MCInstLower &MCIL) {
1344480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
1345480093f4SDimitry Andric 
13460b57cec5SDimitry Andric   // Since PATCHABLE_RET takes the opcode of the return statement as an
13470b57cec5SDimitry Andric   // argument, we use that to emit the correct form of the RET that we want.
13480b57cec5SDimitry Andric   // i.e. when we see this:
13490b57cec5SDimitry Andric   //
13500b57cec5SDimitry Andric   //   PATCHABLE_RET X86::RET ...
13510b57cec5SDimitry Andric   //
13520b57cec5SDimitry Andric   // We should emit the RET followed by sleds.
13530b57cec5SDimitry Andric   //
13540b57cec5SDimitry Andric   //   .p2align 1, ...
13550b57cec5SDimitry Andric   // .Lxray_sled_N:
13560b57cec5SDimitry Andric   //   ret  # or equivalent instruction
13570b57cec5SDimitry Andric   //   # 10 bytes worth of noops
13580b57cec5SDimitry Andric   //
13590b57cec5SDimitry Andric   // This just makes sure that the alignment for the next instruction is 2.
13600b57cec5SDimitry Andric   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1361bdd1243dSDimitry Andric   OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
13625ffd83dbSDimitry Andric   OutStreamer->emitLabel(CurSled);
13630b57cec5SDimitry Andric   unsigned OpCode = MI.getOperand(0).getImm();
13640b57cec5SDimitry Andric   MCInst Ret;
13650b57cec5SDimitry Andric   Ret.setOpcode(OpCode);
1366e8d8bef9SDimitry Andric   for (auto &MO : drop_begin(MI.operands()))
1367*0fca6ea1SDimitry Andric     if (auto Op = MCIL.LowerMachineOperand(&MI, MO); Op.isValid())
1368*0fca6ea1SDimitry Andric       Ret.addOperand(Op);
13695ffd83dbSDimitry Andric   OutStreamer->emitInstruction(Ret, getSubtargetInfo());
13705ffd83dbSDimitry Andric   emitX86Nops(*OutStreamer, 10, Subtarget);
13715ffd83dbSDimitry Andric   recordSled(CurSled, MI, SledKind::FUNCTION_EXIT, 2);
13720b57cec5SDimitry Andric }
13730b57cec5SDimitry Andric 
13740b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
13750b57cec5SDimitry Andric                                              X86MCInstLower &MCIL) {
1376*0fca6ea1SDimitry Andric   MCInst TC;
1377*0fca6ea1SDimitry Andric   TC.setOpcode(convertTailJumpOpcode(MI.getOperand(0).getImm()));
1378*0fca6ea1SDimitry Andric   // Drop the tail jump opcode.
1379*0fca6ea1SDimitry Andric   auto TCOperands = drop_begin(MI.operands());
1380*0fca6ea1SDimitry Andric   bool IsConditional = TC.getOpcode() == X86::JCC_1;
1381*0fca6ea1SDimitry Andric   MCSymbol *FallthroughLabel;
1382*0fca6ea1SDimitry Andric   if (IsConditional) {
1383*0fca6ea1SDimitry Andric     // Rewrite:
1384*0fca6ea1SDimitry Andric     //   je target
1385*0fca6ea1SDimitry Andric     //
1386*0fca6ea1SDimitry Andric     // To:
1387*0fca6ea1SDimitry Andric     //   jne .fallthrough
1388*0fca6ea1SDimitry Andric     //   .p2align 1, ...
1389*0fca6ea1SDimitry Andric     // .Lxray_sled_N:
1390*0fca6ea1SDimitry Andric     //   SLED_CODE
1391*0fca6ea1SDimitry Andric     //   jmp target
1392*0fca6ea1SDimitry Andric     // .fallthrough:
1393*0fca6ea1SDimitry Andric     FallthroughLabel = OutContext.createTempSymbol();
1394*0fca6ea1SDimitry Andric     EmitToStreamer(
1395*0fca6ea1SDimitry Andric         *OutStreamer,
1396*0fca6ea1SDimitry Andric         MCInstBuilder(X86::JCC_1)
1397*0fca6ea1SDimitry Andric             .addExpr(MCSymbolRefExpr::create(FallthroughLabel, OutContext))
1398*0fca6ea1SDimitry Andric             .addImm(X86::GetOppositeBranchCondition(
1399*0fca6ea1SDimitry Andric                 static_cast<X86::CondCode>(MI.getOperand(2).getImm()))));
1400*0fca6ea1SDimitry Andric     TC.setOpcode(X86::JMP_1);
1401*0fca6ea1SDimitry Andric     // Drop the condition code.
1402*0fca6ea1SDimitry Andric     TCOperands = drop_end(TCOperands);
1403*0fca6ea1SDimitry Andric   }
1404*0fca6ea1SDimitry Andric 
1405480093f4SDimitry Andric   NoAutoPaddingScope NoPadScope(*OutStreamer);
1406480093f4SDimitry Andric 
14070b57cec5SDimitry Andric   // Like PATCHABLE_RET, we have the actual instruction in the operands to this
14080b57cec5SDimitry Andric   // instruction so we lower that particular instruction and its operands.
14090b57cec5SDimitry Andric   // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
14100b57cec5SDimitry Andric   // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
14110b57cec5SDimitry Andric   // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
14120b57cec5SDimitry Andric   // tail call much like how we have it in PATCHABLE_RET.
14130b57cec5SDimitry Andric   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1414bdd1243dSDimitry Andric   OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
14155ffd83dbSDimitry Andric   OutStreamer->emitLabel(CurSled);
14160b57cec5SDimitry Andric   auto Target = OutContext.createTempSymbol();
14170b57cec5SDimitry Andric 
14180b57cec5SDimitry Andric   // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
14190b57cec5SDimitry Andric   // an operand (computed as an offset from the jmp instruction).
14200b57cec5SDimitry Andric   // FIXME: Find another less hacky way do force the relative jump.
14215ffd83dbSDimitry Andric   OutStreamer->emitBytes("\xeb\x09");
14225ffd83dbSDimitry Andric   emitX86Nops(*OutStreamer, 9, Subtarget);
14235ffd83dbSDimitry Andric   OutStreamer->emitLabel(Target);
14245ffd83dbSDimitry Andric   recordSled(CurSled, MI, SledKind::TAIL_CALL, 2);
14250b57cec5SDimitry Andric 
14260b57cec5SDimitry Andric   // Before emitting the instruction, add a comment to indicate that this is
14270b57cec5SDimitry Andric   // indeed a tail call.
14280b57cec5SDimitry Andric   OutStreamer->AddComment("TAILCALL");
1429*0fca6ea1SDimitry Andric   for (auto &MO : TCOperands)
1430*0fca6ea1SDimitry Andric     if (auto Op = MCIL.LowerMachineOperand(&MI, MO); Op.isValid())
1431*0fca6ea1SDimitry Andric       TC.addOperand(Op);
14325ffd83dbSDimitry Andric   OutStreamer->emitInstruction(TC, getSubtargetInfo());
1433*0fca6ea1SDimitry Andric 
1434*0fca6ea1SDimitry Andric   if (IsConditional)
1435*0fca6ea1SDimitry Andric     OutStreamer->emitLabel(FallthroughLabel);
14360b57cec5SDimitry Andric }
14370b57cec5SDimitry Andric 
14380b57cec5SDimitry Andric // Returns instruction preceding MBBI in MachineFunction.
14390b57cec5SDimitry Andric // If MBBI is the first instruction of the first basic block, returns null.
14400b57cec5SDimitry Andric static MachineBasicBlock::const_iterator
14410b57cec5SDimitry Andric PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) {
14420b57cec5SDimitry Andric   const MachineBasicBlock *MBB = MBBI->getParent();
14430b57cec5SDimitry Andric   while (MBBI == MBB->begin()) {
14440b57cec5SDimitry Andric     if (MBB == &MBB->getParent()->front())
14450b57cec5SDimitry Andric       return MachineBasicBlock::const_iterator();
14460b57cec5SDimitry Andric     MBB = MBB->getPrevNode();
14470b57cec5SDimitry Andric     MBBI = MBB->end();
14480b57cec5SDimitry Andric   }
14490b57cec5SDimitry Andric   --MBBI;
14500b57cec5SDimitry Andric   return MBBI;
14510b57cec5SDimitry Andric }
14520b57cec5SDimitry Andric 
1453*0fca6ea1SDimitry Andric static unsigned getSrcIdx(const MachineInstr* MI, unsigned SrcIdx) {
1454*0fca6ea1SDimitry Andric   if (X86II::isKMasked(MI->getDesc().TSFlags)) {
1455*0fca6ea1SDimitry Andric     // Skip mask operand.
1456*0fca6ea1SDimitry Andric     ++SrcIdx;
1457*0fca6ea1SDimitry Andric     if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
1458*0fca6ea1SDimitry Andric       // Skip passthru operand.
1459*0fca6ea1SDimitry Andric       ++SrcIdx;
1460*0fca6ea1SDimitry Andric     }
1461*0fca6ea1SDimitry Andric   }
1462*0fca6ea1SDimitry Andric   return SrcIdx;
1463*0fca6ea1SDimitry Andric }
14640b57cec5SDimitry Andric 
1465*0fca6ea1SDimitry Andric static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI,
1466*0fca6ea1SDimitry Andric                                  unsigned SrcOpIdx) {
14670b57cec5SDimitry Andric   const MachineOperand &DstOp = MI->getOperand(0);
1468*0fca6ea1SDimitry Andric   CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg());
14690b57cec5SDimitry Andric 
1470*0fca6ea1SDimitry Andric   // Handle AVX512 MASK/MASXZ write mask comments.
1471*0fca6ea1SDimitry Andric   // MASK: zmmX {%kY}
1472*0fca6ea1SDimitry Andric   // MASKZ: zmmX {%kY} {z}
1473*0fca6ea1SDimitry Andric   if (X86II::isKMasked(MI->getDesc().TSFlags)) {
1474*0fca6ea1SDimitry Andric     const MachineOperand &WriteMaskOp = MI->getOperand(SrcOpIdx - 1);
1475*0fca6ea1SDimitry Andric     StringRef Mask = X86ATTInstPrinter::getRegisterName(WriteMaskOp.getReg());
1476*0fca6ea1SDimitry Andric     CS << " {%" << Mask << "}";
1477*0fca6ea1SDimitry Andric     if (!X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
1478*0fca6ea1SDimitry Andric       CS << " {z}";
1479*0fca6ea1SDimitry Andric     }
1480*0fca6ea1SDimitry Andric   }
1481*0fca6ea1SDimitry Andric }
14820b57cec5SDimitry Andric 
1483*0fca6ea1SDimitry Andric static void printShuffleMask(raw_ostream &CS, StringRef Src1Name,
1484*0fca6ea1SDimitry Andric                              StringRef Src2Name, ArrayRef<int> Mask) {
14850b57cec5SDimitry Andric   // One source operand, fix the mask to print all elements in one span.
1486bdd1243dSDimitry Andric   SmallVector<int, 8> ShuffleMask(Mask);
14870b57cec5SDimitry Andric   if (Src1Name == Src2Name)
14880b57cec5SDimitry Andric     for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
14890b57cec5SDimitry Andric       if (ShuffleMask[i] >= e)
14900b57cec5SDimitry Andric         ShuffleMask[i] -= e;
14910b57cec5SDimitry Andric 
14920b57cec5SDimitry Andric   for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
14930b57cec5SDimitry Andric     if (i != 0)
14940b57cec5SDimitry Andric       CS << ",";
14950b57cec5SDimitry Andric     if (ShuffleMask[i] == SM_SentinelZero) {
14960b57cec5SDimitry Andric       CS << "zero";
14970b57cec5SDimitry Andric       continue;
14980b57cec5SDimitry Andric     }
14990b57cec5SDimitry Andric 
15000b57cec5SDimitry Andric     // Otherwise, it must come from src1 or src2.  Print the span of elements
15010b57cec5SDimitry Andric     // that comes from this src.
15020b57cec5SDimitry Andric     bool isSrc1 = ShuffleMask[i] < (int)e;
15030b57cec5SDimitry Andric     CS << (isSrc1 ? Src1Name : Src2Name) << '[';
15040b57cec5SDimitry Andric 
15050b57cec5SDimitry Andric     bool IsFirst = true;
15060b57cec5SDimitry Andric     while (i != e && ShuffleMask[i] != SM_SentinelZero &&
15070b57cec5SDimitry Andric            (ShuffleMask[i] < (int)e) == isSrc1) {
15080b57cec5SDimitry Andric       if (!IsFirst)
15090b57cec5SDimitry Andric         CS << ',';
15100b57cec5SDimitry Andric       else
15110b57cec5SDimitry Andric         IsFirst = false;
15120b57cec5SDimitry Andric       if (ShuffleMask[i] == SM_SentinelUndef)
15130b57cec5SDimitry Andric         CS << "u";
15140b57cec5SDimitry Andric       else
15150b57cec5SDimitry Andric         CS << ShuffleMask[i] % (int)e;
15160b57cec5SDimitry Andric       ++i;
15170b57cec5SDimitry Andric     }
15180b57cec5SDimitry Andric     CS << ']';
15190b57cec5SDimitry Andric     --i; // For loop increments element #.
15200b57cec5SDimitry Andric   }
1521*0fca6ea1SDimitry Andric }
1522*0fca6ea1SDimitry Andric 
1523*0fca6ea1SDimitry Andric static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
1524*0fca6ea1SDimitry Andric                                      unsigned SrcOp2Idx, ArrayRef<int> Mask) {
1525*0fca6ea1SDimitry Andric   std::string Comment;
1526*0fca6ea1SDimitry Andric 
1527*0fca6ea1SDimitry Andric   const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
1528*0fca6ea1SDimitry Andric   const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
1529*0fca6ea1SDimitry Andric   StringRef Src1Name = SrcOp1.isReg()
1530*0fca6ea1SDimitry Andric                            ? X86ATTInstPrinter::getRegisterName(SrcOp1.getReg())
1531*0fca6ea1SDimitry Andric                            : "mem";
1532*0fca6ea1SDimitry Andric   StringRef Src2Name = SrcOp2.isReg()
1533*0fca6ea1SDimitry Andric                            ? X86ATTInstPrinter::getRegisterName(SrcOp2.getReg())
1534*0fca6ea1SDimitry Andric                            : "mem";
1535*0fca6ea1SDimitry Andric 
1536*0fca6ea1SDimitry Andric   raw_string_ostream CS(Comment);
1537*0fca6ea1SDimitry Andric   printDstRegisterName(CS, MI, SrcOp1Idx);
1538*0fca6ea1SDimitry Andric   CS << " = ";
1539*0fca6ea1SDimitry Andric   printShuffleMask(CS, Src1Name, Src2Name, Mask);
15400b57cec5SDimitry Andric   CS.flush();
15410b57cec5SDimitry Andric 
15420b57cec5SDimitry Andric   return Comment;
15430b57cec5SDimitry Andric }
15440b57cec5SDimitry Andric 
15457a6dacacSDimitry Andric static void printConstant(const APInt &Val, raw_ostream &CS,
15467a6dacacSDimitry Andric                           bool PrintZero = false) {
15470b57cec5SDimitry Andric   if (Val.getBitWidth() <= 64) {
15487a6dacacSDimitry Andric     CS << (PrintZero ? 0ULL : Val.getZExtValue());
15490b57cec5SDimitry Andric   } else {
15500b57cec5SDimitry Andric     // print multi-word constant as (w0,w1)
15510b57cec5SDimitry Andric     CS << "(";
15520b57cec5SDimitry Andric     for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
15530b57cec5SDimitry Andric       if (i > 0)
15540b57cec5SDimitry Andric         CS << ",";
15557a6dacacSDimitry Andric       CS << (PrintZero ? 0ULL : Val.getRawData()[i]);
15560b57cec5SDimitry Andric     }
15570b57cec5SDimitry Andric     CS << ")";
15580b57cec5SDimitry Andric   }
15590b57cec5SDimitry Andric }
15600b57cec5SDimitry Andric 
15617a6dacacSDimitry Andric static void printConstant(const APFloat &Flt, raw_ostream &CS,
15627a6dacacSDimitry Andric                           bool PrintZero = false) {
15630b57cec5SDimitry Andric   SmallString<32> Str;
15647a6dacacSDimitry Andric   // Force scientific notation to distinguish from integers.
15657a6dacacSDimitry Andric   if (PrintZero)
15667a6dacacSDimitry Andric     APFloat::getZero(Flt.getSemantics()).toString(Str, 0, 0);
15677a6dacacSDimitry Andric   else
15680b57cec5SDimitry Andric     Flt.toString(Str, 0, 0);
15690b57cec5SDimitry Andric   CS << Str;
15700b57cec5SDimitry Andric }
15710b57cec5SDimitry Andric 
157206c3fb27SDimitry Andric static void printConstant(const Constant *COp, unsigned BitWidth,
15737a6dacacSDimitry Andric                           raw_ostream &CS, bool PrintZero = false) {
15740b57cec5SDimitry Andric   if (isa<UndefValue>(COp)) {
15750b57cec5SDimitry Andric     CS << "u";
15760b57cec5SDimitry Andric   } else if (auto *CI = dyn_cast<ConstantInt>(COp)) {
15777a6dacacSDimitry Andric     printConstant(CI->getValue(), CS, PrintZero);
15780b57cec5SDimitry Andric   } else if (auto *CF = dyn_cast<ConstantFP>(COp)) {
15797a6dacacSDimitry Andric     printConstant(CF->getValueAPF(), CS, PrintZero);
158006c3fb27SDimitry Andric   } else if (auto *CDS = dyn_cast<ConstantDataSequential>(COp)) {
158106c3fb27SDimitry Andric     Type *EltTy = CDS->getElementType();
158206c3fb27SDimitry Andric     bool IsInteger = EltTy->isIntegerTy();
158306c3fb27SDimitry Andric     bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
158406c3fb27SDimitry Andric     unsigned EltBits = EltTy->getPrimitiveSizeInBits();
158506c3fb27SDimitry Andric     unsigned E = std::min(BitWidth / EltBits, CDS->getNumElements());
15867a6dacacSDimitry Andric     assert((BitWidth % EltBits) == 0 && "Element size mismatch");
158706c3fb27SDimitry Andric     for (unsigned I = 0; I != E; ++I) {
158806c3fb27SDimitry Andric       if (I != 0)
158906c3fb27SDimitry Andric         CS << ",";
159006c3fb27SDimitry Andric       if (IsInteger)
15917a6dacacSDimitry Andric         printConstant(CDS->getElementAsAPInt(I), CS, PrintZero);
159206c3fb27SDimitry Andric       else if (IsFP)
15937a6dacacSDimitry Andric         printConstant(CDS->getElementAsAPFloat(I), CS, PrintZero);
159406c3fb27SDimitry Andric       else
159506c3fb27SDimitry Andric         CS << "?";
159606c3fb27SDimitry Andric     }
15977a6dacacSDimitry Andric   } else if (auto *CV = dyn_cast<ConstantVector>(COp)) {
15987a6dacacSDimitry Andric     unsigned EltBits = CV->getType()->getScalarSizeInBits();
15997a6dacacSDimitry Andric     unsigned E = std::min(BitWidth / EltBits, CV->getNumOperands());
16007a6dacacSDimitry Andric     assert((BitWidth % EltBits) == 0 && "Element size mismatch");
16017a6dacacSDimitry Andric     for (unsigned I = 0; I != E; ++I) {
16027a6dacacSDimitry Andric       if (I != 0)
16037a6dacacSDimitry Andric         CS << ",";
16047a6dacacSDimitry Andric       printConstant(CV->getOperand(I), EltBits, CS, PrintZero);
16057a6dacacSDimitry Andric     }
16060b57cec5SDimitry Andric   } else {
16070b57cec5SDimitry Andric     CS << "?";
16080b57cec5SDimitry Andric   }
16090b57cec5SDimitry Andric }
16100b57cec5SDimitry Andric 
16117a6dacacSDimitry Andric static void printZeroUpperMove(const MachineInstr *MI, MCStreamer &OutStreamer,
16127a6dacacSDimitry Andric                                int SclWidth, int VecWidth,
16137a6dacacSDimitry Andric                                const char *ShuffleComment) {
1614*0fca6ea1SDimitry Andric   unsigned SrcIdx = getSrcIdx(MI, 1);
1615*0fca6ea1SDimitry Andric 
16167a6dacacSDimitry Andric   std::string Comment;
16177a6dacacSDimitry Andric   raw_string_ostream CS(Comment);
1618*0fca6ea1SDimitry Andric   printDstRegisterName(CS, MI, SrcIdx);
1619*0fca6ea1SDimitry Andric   CS << " = ";
16207a6dacacSDimitry Andric 
1621*0fca6ea1SDimitry Andric   if (auto *C = X86::getConstantFromPool(*MI, SrcIdx)) {
16227a6dacacSDimitry Andric     CS << "[";
16237a6dacacSDimitry Andric     printConstant(C, SclWidth, CS);
16247a6dacacSDimitry Andric     for (int I = 1, E = VecWidth / SclWidth; I < E; ++I) {
16257a6dacacSDimitry Andric       CS << ",";
16267a6dacacSDimitry Andric       printConstant(C, SclWidth, CS, true);
16277a6dacacSDimitry Andric     }
16287a6dacacSDimitry Andric     CS << "]";
16297a6dacacSDimitry Andric     OutStreamer.AddComment(CS.str());
16307a6dacacSDimitry Andric     return; // early-out
16317a6dacacSDimitry Andric   }
16327a6dacacSDimitry Andric 
16337a6dacacSDimitry Andric   // We didn't find a constant load, fallback to a shuffle mask decode.
16347a6dacacSDimitry Andric   CS << ShuffleComment;
16357a6dacacSDimitry Andric   OutStreamer.AddComment(CS.str());
16367a6dacacSDimitry Andric }
16377a6dacacSDimitry Andric 
16387a6dacacSDimitry Andric static void printBroadcast(const MachineInstr *MI, MCStreamer &OutStreamer,
16397a6dacacSDimitry Andric                            int Repeats, int BitWidth) {
1640*0fca6ea1SDimitry Andric   unsigned SrcIdx = getSrcIdx(MI, 1);
1641*0fca6ea1SDimitry Andric   if (auto *C = X86::getConstantFromPool(*MI, SrcIdx)) {
16427a6dacacSDimitry Andric     std::string Comment;
16437a6dacacSDimitry Andric     raw_string_ostream CS(Comment);
1644*0fca6ea1SDimitry Andric     printDstRegisterName(CS, MI, SrcIdx);
1645*0fca6ea1SDimitry Andric     CS << " = [";
16467a6dacacSDimitry Andric     for (int l = 0; l != Repeats; ++l) {
16477a6dacacSDimitry Andric       if (l != 0)
16487a6dacacSDimitry Andric         CS << ",";
16497a6dacacSDimitry Andric       printConstant(C, BitWidth, CS);
16507a6dacacSDimitry Andric     }
16517a6dacacSDimitry Andric     CS << "]";
16527a6dacacSDimitry Andric     OutStreamer.AddComment(CS.str());
16537a6dacacSDimitry Andric   }
16547a6dacacSDimitry Andric }
16557a6dacacSDimitry Andric 
1656*0fca6ea1SDimitry Andric static bool printExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1657*0fca6ea1SDimitry Andric                         int SrcEltBits, int DstEltBits, bool IsSext) {
1658*0fca6ea1SDimitry Andric   unsigned SrcIdx = getSrcIdx(MI, 1);
1659*0fca6ea1SDimitry Andric   auto *C = X86::getConstantFromPool(*MI, SrcIdx);
1660*0fca6ea1SDimitry Andric   if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) {
1661*0fca6ea1SDimitry Andric     if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
1662*0fca6ea1SDimitry Andric       int NumElts = CDS->getNumElements();
1663*0fca6ea1SDimitry Andric       std::string Comment;
1664*0fca6ea1SDimitry Andric       raw_string_ostream CS(Comment);
1665*0fca6ea1SDimitry Andric       printDstRegisterName(CS, MI, SrcIdx);
1666*0fca6ea1SDimitry Andric       CS << " = [";
1667*0fca6ea1SDimitry Andric       for (int i = 0; i != NumElts; ++i) {
1668*0fca6ea1SDimitry Andric         if (i != 0)
1669*0fca6ea1SDimitry Andric           CS << ",";
1670*0fca6ea1SDimitry Andric         if (CDS->getElementType()->isIntegerTy()) {
1671*0fca6ea1SDimitry Andric           APInt Elt = CDS->getElementAsAPInt(i);
1672*0fca6ea1SDimitry Andric           Elt = IsSext ? Elt.sext(DstEltBits) : Elt.zext(DstEltBits);
1673*0fca6ea1SDimitry Andric           printConstant(Elt, CS);
1674*0fca6ea1SDimitry Andric         } else
1675*0fca6ea1SDimitry Andric           CS << "?";
1676*0fca6ea1SDimitry Andric       }
1677*0fca6ea1SDimitry Andric       CS << "]";
1678*0fca6ea1SDimitry Andric       OutStreamer.AddComment(CS.str());
1679*0fca6ea1SDimitry Andric       return true;
1680*0fca6ea1SDimitry Andric     }
1681*0fca6ea1SDimitry Andric   }
1682*0fca6ea1SDimitry Andric 
1683*0fca6ea1SDimitry Andric   return false;
1684*0fca6ea1SDimitry Andric }
1685*0fca6ea1SDimitry Andric static void printSignExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1686*0fca6ea1SDimitry Andric                             int SrcEltBits, int DstEltBits) {
1687*0fca6ea1SDimitry Andric   printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, true);
1688*0fca6ea1SDimitry Andric }
1689*0fca6ea1SDimitry Andric static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1690*0fca6ea1SDimitry Andric                             int SrcEltBits, int DstEltBits) {
1691*0fca6ea1SDimitry Andric   if (printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, false))
1692*0fca6ea1SDimitry Andric     return;
1693*0fca6ea1SDimitry Andric 
1694*0fca6ea1SDimitry Andric   // We didn't find a constant load, fallback to a shuffle mask decode.
1695*0fca6ea1SDimitry Andric   std::string Comment;
1696*0fca6ea1SDimitry Andric   raw_string_ostream CS(Comment);
1697*0fca6ea1SDimitry Andric   printDstRegisterName(CS, MI, getSrcIdx(MI, 1));
1698*0fca6ea1SDimitry Andric   CS << " = ";
1699*0fca6ea1SDimitry Andric 
1700*0fca6ea1SDimitry Andric   SmallVector<int> Mask;
1701*0fca6ea1SDimitry Andric   unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1702*0fca6ea1SDimitry Andric   assert((Width % DstEltBits) == 0 && (DstEltBits % SrcEltBits) == 0 &&
1703*0fca6ea1SDimitry Andric          "Illegal extension ratio");
1704*0fca6ea1SDimitry Andric   DecodeZeroExtendMask(SrcEltBits, DstEltBits, Width / DstEltBits, false, Mask);
1705*0fca6ea1SDimitry Andric   printShuffleMask(CS, "mem", "", Mask);
1706*0fca6ea1SDimitry Andric 
1707*0fca6ea1SDimitry Andric   OutStreamer.AddComment(CS.str());
1708*0fca6ea1SDimitry Andric }
1709*0fca6ea1SDimitry Andric 
17100b57cec5SDimitry Andric void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
17110b57cec5SDimitry Andric   assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
171206c3fb27SDimitry Andric   assert((getSubtarget().isOSWindows() || TM.getTargetTriple().isUEFI()) &&
171306c3fb27SDimitry Andric          "SEH_ instruction Windows and UEFI only");
17140b57cec5SDimitry Andric 
17150b57cec5SDimitry Andric   // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
17160b57cec5SDimitry Andric   if (EmitFPOData) {
17170b57cec5SDimitry Andric     X86TargetStreamer *XTS =
17180b57cec5SDimitry Andric         static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
17190b57cec5SDimitry Andric     switch (MI->getOpcode()) {
17200b57cec5SDimitry Andric     case X86::SEH_PushReg:
17210b57cec5SDimitry Andric       XTS->emitFPOPushReg(MI->getOperand(0).getImm());
17220b57cec5SDimitry Andric       break;
17230b57cec5SDimitry Andric     case X86::SEH_StackAlloc:
17240b57cec5SDimitry Andric       XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());
17250b57cec5SDimitry Andric       break;
17260b57cec5SDimitry Andric     case X86::SEH_StackAlign:
17270b57cec5SDimitry Andric       XTS->emitFPOStackAlign(MI->getOperand(0).getImm());
17280b57cec5SDimitry Andric       break;
17290b57cec5SDimitry Andric     case X86::SEH_SetFrame:
17300b57cec5SDimitry Andric       assert(MI->getOperand(1).getImm() == 0 &&
17310b57cec5SDimitry Andric              ".cv_fpo_setframe takes no offset");
17320b57cec5SDimitry Andric       XTS->emitFPOSetFrame(MI->getOperand(0).getImm());
17330b57cec5SDimitry Andric       break;
17340b57cec5SDimitry Andric     case X86::SEH_EndPrologue:
17350b57cec5SDimitry Andric       XTS->emitFPOEndPrologue();
17360b57cec5SDimitry Andric       break;
17370b57cec5SDimitry Andric     case X86::SEH_SaveReg:
17380b57cec5SDimitry Andric     case X86::SEH_SaveXMM:
17390b57cec5SDimitry Andric     case X86::SEH_PushFrame:
17400b57cec5SDimitry Andric       llvm_unreachable("SEH_ directive incompatible with FPO");
17410b57cec5SDimitry Andric       break;
17420b57cec5SDimitry Andric     default:
17430b57cec5SDimitry Andric       llvm_unreachable("expected SEH_ instruction");
17440b57cec5SDimitry Andric     }
17450b57cec5SDimitry Andric     return;
17460b57cec5SDimitry Andric   }
17470b57cec5SDimitry Andric 
17480b57cec5SDimitry Andric   // Otherwise, use the .seh_ directives for all other Windows platforms.
17490b57cec5SDimitry Andric   switch (MI->getOpcode()) {
17500b57cec5SDimitry Andric   case X86::SEH_PushReg:
175181ad6265SDimitry Andric     OutStreamer->emitWinCFIPushReg(MI->getOperand(0).getImm());
17520b57cec5SDimitry Andric     break;
17530b57cec5SDimitry Andric 
17540b57cec5SDimitry Andric   case X86::SEH_SaveReg:
175581ad6265SDimitry Andric     OutStreamer->emitWinCFISaveReg(MI->getOperand(0).getImm(),
17560b57cec5SDimitry Andric                                    MI->getOperand(1).getImm());
17570b57cec5SDimitry Andric     break;
17580b57cec5SDimitry Andric 
17590b57cec5SDimitry Andric   case X86::SEH_SaveXMM:
176081ad6265SDimitry Andric     OutStreamer->emitWinCFISaveXMM(MI->getOperand(0).getImm(),
17610b57cec5SDimitry Andric                                    MI->getOperand(1).getImm());
17620b57cec5SDimitry Andric     break;
17630b57cec5SDimitry Andric 
17640b57cec5SDimitry Andric   case X86::SEH_StackAlloc:
176581ad6265SDimitry Andric     OutStreamer->emitWinCFIAllocStack(MI->getOperand(0).getImm());
17660b57cec5SDimitry Andric     break;
17670b57cec5SDimitry Andric 
17680b57cec5SDimitry Andric   case X86::SEH_SetFrame:
176981ad6265SDimitry Andric     OutStreamer->emitWinCFISetFrame(MI->getOperand(0).getImm(),
17700b57cec5SDimitry Andric                                     MI->getOperand(1).getImm());
17710b57cec5SDimitry Andric     break;
17720b57cec5SDimitry Andric 
17730b57cec5SDimitry Andric   case X86::SEH_PushFrame:
177481ad6265SDimitry Andric     OutStreamer->emitWinCFIPushFrame(MI->getOperand(0).getImm());
17750b57cec5SDimitry Andric     break;
17760b57cec5SDimitry Andric 
17770b57cec5SDimitry Andric   case X86::SEH_EndPrologue:
177881ad6265SDimitry Andric     OutStreamer->emitWinCFIEndProlog();
17790b57cec5SDimitry Andric     break;
17800b57cec5SDimitry Andric 
17810b57cec5SDimitry Andric   default:
17820b57cec5SDimitry Andric     llvm_unreachable("expected SEH_ instruction");
17830b57cec5SDimitry Andric   }
17840b57cec5SDimitry Andric }
17850b57cec5SDimitry Andric 
17865ffd83dbSDimitry Andric static void addConstantComments(const MachineInstr *MI,
17875ffd83dbSDimitry Andric                                 MCStreamer &OutStreamer) {
17885ffd83dbSDimitry Andric   switch (MI->getOpcode()) {
17895ffd83dbSDimitry Andric   // Lower PSHUFB and VPERMILP normally but add a comment if we can find
17905ffd83dbSDimitry Andric   // a constant shuffle mask. We won't be able to do this at the MC layer
17915ffd83dbSDimitry Andric   // because the mask isn't an immediate.
17925ffd83dbSDimitry Andric   case X86::PSHUFBrm:
17935ffd83dbSDimitry Andric   case X86::VPSHUFBrm:
17945ffd83dbSDimitry Andric   case X86::VPSHUFBYrm:
17955ffd83dbSDimitry Andric   case X86::VPSHUFBZ128rm:
17965ffd83dbSDimitry Andric   case X86::VPSHUFBZ128rmk:
17975ffd83dbSDimitry Andric   case X86::VPSHUFBZ128rmkz:
17985ffd83dbSDimitry Andric   case X86::VPSHUFBZ256rm:
17995ffd83dbSDimitry Andric   case X86::VPSHUFBZ256rmk:
18005ffd83dbSDimitry Andric   case X86::VPSHUFBZ256rmkz:
18015ffd83dbSDimitry Andric   case X86::VPSHUFBZrm:
18025ffd83dbSDimitry Andric   case X86::VPSHUFBZrmk:
18035ffd83dbSDimitry Andric   case X86::VPSHUFBZrmkz: {
1804*0fca6ea1SDimitry Andric     unsigned SrcIdx = getSrcIdx(MI, 1);
18057a6dacacSDimitry Andric     if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {
1806*0fca6ea1SDimitry Andric       unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
18075ffd83dbSDimitry Andric       SmallVector<int, 64> Mask;
18085ffd83dbSDimitry Andric       DecodePSHUFBMask(C, Width, Mask);
18095ffd83dbSDimitry Andric       if (!Mask.empty())
18105ffd83dbSDimitry Andric         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
18115ffd83dbSDimitry Andric     }
18125ffd83dbSDimitry Andric     break;
18135ffd83dbSDimitry Andric   }
18145ffd83dbSDimitry Andric 
18155ffd83dbSDimitry Andric   case X86::VPERMILPSrm:
18165ffd83dbSDimitry Andric   case X86::VPERMILPSYrm:
18175ffd83dbSDimitry Andric   case X86::VPERMILPSZ128rm:
18185ffd83dbSDimitry Andric   case X86::VPERMILPSZ128rmk:
18195ffd83dbSDimitry Andric   case X86::VPERMILPSZ128rmkz:
18205ffd83dbSDimitry Andric   case X86::VPERMILPSZ256rm:
18215ffd83dbSDimitry Andric   case X86::VPERMILPSZ256rmk:
18225ffd83dbSDimitry Andric   case X86::VPERMILPSZ256rmkz:
18235ffd83dbSDimitry Andric   case X86::VPERMILPSZrm:
18245ffd83dbSDimitry Andric   case X86::VPERMILPSZrmk:
1825*0fca6ea1SDimitry Andric   case X86::VPERMILPSZrmkz: {
1826*0fca6ea1SDimitry Andric     unsigned SrcIdx = getSrcIdx(MI, 1);
1827*0fca6ea1SDimitry Andric     if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {
1828*0fca6ea1SDimitry Andric       unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1829*0fca6ea1SDimitry Andric       SmallVector<int, 16> Mask;
1830*0fca6ea1SDimitry Andric       DecodeVPERMILPMask(C, 32, Width, Mask);
1831*0fca6ea1SDimitry Andric       if (!Mask.empty())
1832*0fca6ea1SDimitry Andric         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
1833*0fca6ea1SDimitry Andric     }
1834*0fca6ea1SDimitry Andric     break;
1835*0fca6ea1SDimitry Andric   }
18365ffd83dbSDimitry Andric   case X86::VPERMILPDrm:
18375ffd83dbSDimitry Andric   case X86::VPERMILPDYrm:
18385ffd83dbSDimitry Andric   case X86::VPERMILPDZ128rm:
18395ffd83dbSDimitry Andric   case X86::VPERMILPDZ128rmk:
18405ffd83dbSDimitry Andric   case X86::VPERMILPDZ128rmkz:
18415ffd83dbSDimitry Andric   case X86::VPERMILPDZ256rm:
18425ffd83dbSDimitry Andric   case X86::VPERMILPDZ256rmk:
18435ffd83dbSDimitry Andric   case X86::VPERMILPDZ256rmkz:
18445ffd83dbSDimitry Andric   case X86::VPERMILPDZrm:
18455ffd83dbSDimitry Andric   case X86::VPERMILPDZrmk:
18465ffd83dbSDimitry Andric   case X86::VPERMILPDZrmkz: {
1847*0fca6ea1SDimitry Andric     unsigned SrcIdx = getSrcIdx(MI, 1);
18487a6dacacSDimitry Andric     if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {
1849*0fca6ea1SDimitry Andric       unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
18505ffd83dbSDimitry Andric       SmallVector<int, 16> Mask;
1851*0fca6ea1SDimitry Andric       DecodeVPERMILPMask(C, 64, Width, Mask);
18525ffd83dbSDimitry Andric       if (!Mask.empty())
18535ffd83dbSDimitry Andric         OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
18545ffd83dbSDimitry Andric     }
18555ffd83dbSDimitry Andric     break;
18565ffd83dbSDimitry Andric   }
18575ffd83dbSDimitry Andric 
18585ffd83dbSDimitry Andric   case X86::VPERMIL2PDrm:
18595ffd83dbSDimitry Andric   case X86::VPERMIL2PSrm:
18605ffd83dbSDimitry Andric   case X86::VPERMIL2PDYrm:
18615ffd83dbSDimitry Andric   case X86::VPERMIL2PSYrm: {
18625ffd83dbSDimitry Andric     assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
18635ffd83dbSDimitry Andric            "Unexpected number of operands!");
18645ffd83dbSDimitry Andric 
18655ffd83dbSDimitry Andric     const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
18665ffd83dbSDimitry Andric     if (!CtrlOp.isImm())
18675ffd83dbSDimitry Andric       break;
18685ffd83dbSDimitry Andric 
18695ffd83dbSDimitry Andric     unsigned ElSize;
18705ffd83dbSDimitry Andric     switch (MI->getOpcode()) {
18715ffd83dbSDimitry Andric     default: llvm_unreachable("Invalid opcode");
18725ffd83dbSDimitry Andric     case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
18735ffd83dbSDimitry Andric     case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
18745ffd83dbSDimitry Andric     }
18755ffd83dbSDimitry Andric 
18767a6dacacSDimitry Andric     if (auto *C = X86::getConstantFromPool(*MI, 3)) {
1877*0fca6ea1SDimitry Andric       unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
18785ffd83dbSDimitry Andric       SmallVector<int, 16> Mask;
18795ffd83dbSDimitry Andric       DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);
18805ffd83dbSDimitry Andric       if (!Mask.empty())
18815ffd83dbSDimitry Andric         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
18825ffd83dbSDimitry Andric     }
18835ffd83dbSDimitry Andric     break;
18845ffd83dbSDimitry Andric   }
18855ffd83dbSDimitry Andric 
18865ffd83dbSDimitry Andric   case X86::VPPERMrrm: {
18877a6dacacSDimitry Andric     if (auto *C = X86::getConstantFromPool(*MI, 3)) {
1888*0fca6ea1SDimitry Andric       unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
18895ffd83dbSDimitry Andric       SmallVector<int, 16> Mask;
18905ffd83dbSDimitry Andric       DecodeVPPERMMask(C, Width, Mask);
18915ffd83dbSDimitry Andric       if (!Mask.empty())
18925ffd83dbSDimitry Andric         OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
18935ffd83dbSDimitry Andric     }
18945ffd83dbSDimitry Andric     break;
18955ffd83dbSDimitry Andric   }
18965ffd83dbSDimitry Andric 
18975ffd83dbSDimitry Andric   case X86::MMX_MOVQ64rm: {
18987a6dacacSDimitry Andric     if (auto *C = X86::getConstantFromPool(*MI, 1)) {
18995ffd83dbSDimitry Andric       std::string Comment;
19005ffd83dbSDimitry Andric       raw_string_ostream CS(Comment);
19015ffd83dbSDimitry Andric       const MachineOperand &DstOp = MI->getOperand(0);
19025ffd83dbSDimitry Andric       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
19035ffd83dbSDimitry Andric       if (auto *CF = dyn_cast<ConstantFP>(C)) {
1904fe6060f1SDimitry Andric         CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);
19055ffd83dbSDimitry Andric         OutStreamer.AddComment(CS.str());
19065ffd83dbSDimitry Andric       }
19075ffd83dbSDimitry Andric     }
19085ffd83dbSDimitry Andric     break;
19095ffd83dbSDimitry Andric   }
19105ffd83dbSDimitry Andric 
1911*0fca6ea1SDimitry Andric #define INSTR_CASE(Prefix, Instr, Suffix, Postfix)                             \
1912*0fca6ea1SDimitry Andric   case X86::Prefix##Instr##Suffix##rm##Postfix:
1913*0fca6ea1SDimitry Andric 
1914*0fca6ea1SDimitry Andric #define CASE_ARITH_RM(Instr)                                                   \
1915*0fca6ea1SDimitry Andric   INSTR_CASE(, Instr, , )   /* SSE */                                          \
1916*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, , )  /* AVX-128 */                                      \
1917*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Y, ) /* AVX-256 */                                      \
1918*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z128, )                                                 \
1919*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z128, k)                                                \
1920*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z128, kz)                                               \
1921*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z256, )                                                 \
1922*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z256, k)                                                \
1923*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z256, kz)                                               \
1924*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z, )                                                    \
1925*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z, k)                                                   \
1926*0fca6ea1SDimitry Andric   INSTR_CASE(V, Instr, Z, kz)
1927*0fca6ea1SDimitry Andric 
1928*0fca6ea1SDimitry Andric     // TODO: Add additional instructions when useful.
1929*0fca6ea1SDimitry Andric     CASE_ARITH_RM(PMADDUBSW) {
1930*0fca6ea1SDimitry Andric       unsigned SrcIdx = getSrcIdx(MI, 1);
1931*0fca6ea1SDimitry Andric       if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {
1932*0fca6ea1SDimitry Andric         if (C->getType()->getScalarSizeInBits() == 8) {
1933*0fca6ea1SDimitry Andric           std::string Comment;
1934*0fca6ea1SDimitry Andric           raw_string_ostream CS(Comment);
1935*0fca6ea1SDimitry Andric           unsigned VectorWidth =
1936*0fca6ea1SDimitry Andric               X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1937*0fca6ea1SDimitry Andric           CS << "[";
1938*0fca6ea1SDimitry Andric           printConstant(C, VectorWidth, CS);
1939*0fca6ea1SDimitry Andric           CS << "]";
1940*0fca6ea1SDimitry Andric           OutStreamer.AddComment(CS.str());
1941*0fca6ea1SDimitry Andric         }
1942*0fca6ea1SDimitry Andric       }
1943*0fca6ea1SDimitry Andric       break;
1944*0fca6ea1SDimitry Andric     }
1945*0fca6ea1SDimitry Andric 
1946*0fca6ea1SDimitry Andric     CASE_ARITH_RM(PMADDWD)
1947*0fca6ea1SDimitry Andric     CASE_ARITH_RM(PMULLW)
1948*0fca6ea1SDimitry Andric     CASE_ARITH_RM(PMULHW)
1949*0fca6ea1SDimitry Andric     CASE_ARITH_RM(PMULHUW)
1950*0fca6ea1SDimitry Andric     CASE_ARITH_RM(PMULHRSW) {
1951*0fca6ea1SDimitry Andric       unsigned SrcIdx = getSrcIdx(MI, 1);
1952*0fca6ea1SDimitry Andric       if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {
1953*0fca6ea1SDimitry Andric         if (C->getType()->getScalarSizeInBits() == 16) {
1954*0fca6ea1SDimitry Andric           std::string Comment;
1955*0fca6ea1SDimitry Andric           raw_string_ostream CS(Comment);
1956*0fca6ea1SDimitry Andric           unsigned VectorWidth =
1957*0fca6ea1SDimitry Andric               X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1958*0fca6ea1SDimitry Andric           CS << "[";
1959*0fca6ea1SDimitry Andric           printConstant(C, VectorWidth, CS);
1960*0fca6ea1SDimitry Andric           CS << "]";
1961*0fca6ea1SDimitry Andric           OutStreamer.AddComment(CS.str());
1962*0fca6ea1SDimitry Andric         }
1963*0fca6ea1SDimitry Andric       }
1964*0fca6ea1SDimitry Andric       break;
1965*0fca6ea1SDimitry Andric     }
1966*0fca6ea1SDimitry Andric 
1967*0fca6ea1SDimitry Andric #define MASK_AVX512_CASE(Instr)                                                \
1968*0fca6ea1SDimitry Andric   case Instr:                                                                  \
1969*0fca6ea1SDimitry Andric   case Instr##k:                                                               \
1970*0fca6ea1SDimitry Andric   case Instr##kz:
1971*0fca6ea1SDimitry Andric 
19727a6dacacSDimitry Andric   case X86::MOVSDrm:
19737a6dacacSDimitry Andric   case X86::VMOVSDrm:
1974*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VMOVSDZrm)
19757a6dacacSDimitry Andric   case X86::MOVSDrm_alt:
19767a6dacacSDimitry Andric   case X86::VMOVSDrm_alt:
19777a6dacacSDimitry Andric   case X86::VMOVSDZrm_alt:
19787a6dacacSDimitry Andric   case X86::MOVQI2PQIrm:
19797a6dacacSDimitry Andric   case X86::VMOVQI2PQIrm:
19807a6dacacSDimitry Andric   case X86::VMOVQI2PQIZrm:
19817a6dacacSDimitry Andric     printZeroUpperMove(MI, OutStreamer, 64, 128, "mem[0],zero");
19827a6dacacSDimitry Andric     break;
19837a6dacacSDimitry Andric 
1984*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VMOVSHZrm)
1985*0fca6ea1SDimitry Andric   case X86::VMOVSHZrm_alt:
1986*0fca6ea1SDimitry Andric     printZeroUpperMove(MI, OutStreamer, 16, 128,
1987*0fca6ea1SDimitry Andric                        "mem[0],zero,zero,zero,zero,zero,zero,zero");
1988*0fca6ea1SDimitry Andric     break;
1989*0fca6ea1SDimitry Andric 
19907a6dacacSDimitry Andric   case X86::MOVSSrm:
19917a6dacacSDimitry Andric   case X86::VMOVSSrm:
1992*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VMOVSSZrm)
19937a6dacacSDimitry Andric   case X86::MOVSSrm_alt:
19947a6dacacSDimitry Andric   case X86::VMOVSSrm_alt:
19957a6dacacSDimitry Andric   case X86::VMOVSSZrm_alt:
19967a6dacacSDimitry Andric   case X86::MOVDI2PDIrm:
19977a6dacacSDimitry Andric   case X86::VMOVDI2PDIrm:
19987a6dacacSDimitry Andric   case X86::VMOVDI2PDIZrm:
19997a6dacacSDimitry Andric     printZeroUpperMove(MI, OutStreamer, 32, 128, "mem[0],zero,zero,zero");
20007a6dacacSDimitry Andric     break;
20017a6dacacSDimitry Andric 
20025ffd83dbSDimitry Andric #define MOV_CASE(Prefix, Suffix)                                               \
20035ffd83dbSDimitry Andric   case X86::Prefix##MOVAPD##Suffix##rm:                                        \
20045ffd83dbSDimitry Andric   case X86::Prefix##MOVAPS##Suffix##rm:                                        \
20055ffd83dbSDimitry Andric   case X86::Prefix##MOVUPD##Suffix##rm:                                        \
20065ffd83dbSDimitry Andric   case X86::Prefix##MOVUPS##Suffix##rm:                                        \
20075ffd83dbSDimitry Andric   case X86::Prefix##MOVDQA##Suffix##rm:                                        \
20085ffd83dbSDimitry Andric   case X86::Prefix##MOVDQU##Suffix##rm:
20095ffd83dbSDimitry Andric 
2010*0fca6ea1SDimitry Andric #define MOV_AVX512_CASE(Suffix, Postfix)                                       \
2011*0fca6ea1SDimitry Andric   case X86::VMOVDQA64##Suffix##rm##Postfix:                                    \
2012*0fca6ea1SDimitry Andric   case X86::VMOVDQA32##Suffix##rm##Postfix:                                    \
2013*0fca6ea1SDimitry Andric   case X86::VMOVDQU64##Suffix##rm##Postfix:                                    \
2014*0fca6ea1SDimitry Andric   case X86::VMOVDQU32##Suffix##rm##Postfix:                                    \
2015*0fca6ea1SDimitry Andric   case X86::VMOVDQU16##Suffix##rm##Postfix:                                    \
2016*0fca6ea1SDimitry Andric   case X86::VMOVDQU8##Suffix##rm##Postfix:                                     \
2017*0fca6ea1SDimitry Andric   case X86::VMOVAPS##Suffix##rm##Postfix:                                      \
2018*0fca6ea1SDimitry Andric   case X86::VMOVAPD##Suffix##rm##Postfix:                                      \
2019*0fca6ea1SDimitry Andric   case X86::VMOVUPS##Suffix##rm##Postfix:                                      \
2020*0fca6ea1SDimitry Andric   case X86::VMOVUPD##Suffix##rm##Postfix:
20215ffd83dbSDimitry Andric 
20225f757f3fSDimitry Andric #define CASE_128_MOV_RM()                                                      \
20235f757f3fSDimitry Andric   MOV_CASE(, )   /* SSE */                                                     \
20245f757f3fSDimitry Andric   MOV_CASE(V, )  /* AVX-128 */                                                 \
2025*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z128, )                                                      \
2026*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z128, k)                                                     \
2027*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z128, kz)
20285f757f3fSDimitry Andric 
20295f757f3fSDimitry Andric #define CASE_256_MOV_RM()                                                      \
20305f757f3fSDimitry Andric   MOV_CASE(V, Y) /* AVX-256 */                                                 \
2031*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z256, )                                                      \
2032*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z256, k)                                                     \
2033*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z256, kz)                                                    \
20345f757f3fSDimitry Andric 
20355f757f3fSDimitry Andric #define CASE_512_MOV_RM()                                                      \
2036*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z, )                                                         \
2037*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z, k)                                                        \
2038*0fca6ea1SDimitry Andric   MOV_AVX512_CASE(Z, kz)                                                       \
20395f757f3fSDimitry Andric 
20405ffd83dbSDimitry Andric     // For loads from a constant pool to a vector register, print the constant
20415ffd83dbSDimitry Andric     // loaded.
20427a6dacacSDimitry Andric     CASE_128_MOV_RM()
20437a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 1, 128);
20447a6dacacSDimitry Andric     break;
20457a6dacacSDimitry Andric     CASE_256_MOV_RM()
20467a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 1, 256);
20477a6dacacSDimitry Andric     break;
20487a6dacacSDimitry Andric     CASE_512_MOV_RM()
20497a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 1, 512);
20507a6dacacSDimitry Andric     break;
20515f757f3fSDimitry Andric   case X86::VBROADCASTF128rm:
20525f757f3fSDimitry Andric   case X86::VBROADCASTI128rm:
2053*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTF32X4Z256rm)
2054*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTF64X2Z128rm)
2055*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTI32X4Z256rm)
2056*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTI64X2Z128rm)
20577a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 2, 128);
20587a6dacacSDimitry Andric     break;
2059*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTF32X4rm)
2060*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTF64X2rm)
2061*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTI32X4rm)
2062*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTI64X2rm)
20637a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 4, 128);
20647a6dacacSDimitry Andric     break;
2065*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTF32X8rm)
2066*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTF64X4rm)
2067*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTI32X8rm)
2068*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTI64X4rm)
20697a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 2, 256);
20705ffd83dbSDimitry Andric     break;
20715ffd83dbSDimitry Andric 
20727a6dacacSDimitry Andric   // For broadcast loads from a constant pool to a vector register, repeatedly
20737a6dacacSDimitry Andric   // print the constant loaded.
20745ffd83dbSDimitry Andric   case X86::MOVDDUPrm:
20755ffd83dbSDimitry Andric   case X86::VMOVDDUPrm:
2076*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VMOVDDUPZ128rm)
20777a6dacacSDimitry Andric   case X86::VPBROADCASTQrm:
2078*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTQZ128rm)
20797a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 2, 64);
20807a6dacacSDimitry Andric     break;
20815ffd83dbSDimitry Andric   case X86::VBROADCASTSDYrm:
2082*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTSDZ256rm)
20835ffd83dbSDimitry Andric   case X86::VPBROADCASTQYrm:
2084*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTQZ256rm)
20857a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 4, 64);
20867a6dacacSDimitry Andric     break;
2087*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTSDZrm)
2088*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTQZrm)
20897a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 8, 64);
20907a6dacacSDimitry Andric     break;
20917a6dacacSDimitry Andric   case X86::VBROADCASTSSrm:
2092*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTSSZ128rm)
20937a6dacacSDimitry Andric   case X86::VPBROADCASTDrm:
2094*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTDZ128rm)
20957a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 4, 32);
20967a6dacacSDimitry Andric     break;
20977a6dacacSDimitry Andric   case X86::VBROADCASTSSYrm:
2098*0fca6ea1SDimitry Andric     MASK_AVX512_CASE(X86::VBROADCASTSSZ256rm)
20997a6dacacSDimitry Andric   case X86::VPBROADCASTDYrm:
2100*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTDZ256rm)
21017a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 8, 32);
21027a6dacacSDimitry Andric     break;
2103*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VBROADCASTSSZrm)
2104*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTDZrm)
21057a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 16, 32);
21067a6dacacSDimitry Andric     break;
21075ffd83dbSDimitry Andric   case X86::VPBROADCASTWrm:
2108*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTWZ128rm)
21097a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 8, 16);
21107a6dacacSDimitry Andric     break;
21117a6dacacSDimitry Andric   case X86::VPBROADCASTWYrm:
2112*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTWZ256rm)
21137a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 16, 16);
21147a6dacacSDimitry Andric     break;
2115*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTWZrm)
21167a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 32, 16);
21177a6dacacSDimitry Andric     break;
21187a6dacacSDimitry Andric   case X86::VPBROADCASTBrm:
2119*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTBZ128rm)
21207a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 16, 8);
21217a6dacacSDimitry Andric     break;
21227a6dacacSDimitry Andric   case X86::VPBROADCASTBYrm:
2123*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTBZ256rm)
21247a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 32, 8);
21257a6dacacSDimitry Andric     break;
2126*0fca6ea1SDimitry Andric   MASK_AVX512_CASE(X86::VPBROADCASTBZrm)
21277a6dacacSDimitry Andric     printBroadcast(MI, OutStreamer, 64, 8);
21287a6dacacSDimitry Andric     break;
2129*0fca6ea1SDimitry Andric 
2130*0fca6ea1SDimitry Andric #define MOVX_CASE(Prefix, Ext, Type, Suffix, Postfix)                          \
2131*0fca6ea1SDimitry Andric   case X86::Prefix##PMOV##Ext##Type##Suffix##rm##Postfix:
2132*0fca6ea1SDimitry Andric 
2133*0fca6ea1SDimitry Andric #define CASE_MOVX_RM(Ext, Type)                                                \
2134*0fca6ea1SDimitry Andric   MOVX_CASE(, Ext, Type, , )                                                   \
2135*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, , )                                                  \
2136*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Y, )                                                 \
2137*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z128, )                                              \
2138*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z128, k )                                            \
2139*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z128, kz )                                           \
2140*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z256, )                                              \
2141*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z256, k )                                            \
2142*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z256, kz )                                           \
2143*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z, )                                                 \
2144*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z, k )                                               \
2145*0fca6ea1SDimitry Andric   MOVX_CASE(V, Ext, Type, Z, kz )
2146*0fca6ea1SDimitry Andric 
2147*0fca6ea1SDimitry Andric     CASE_MOVX_RM(SX, BD)
2148*0fca6ea1SDimitry Andric     printSignExtend(MI, OutStreamer, 8, 32);
2149*0fca6ea1SDimitry Andric     break;
2150*0fca6ea1SDimitry Andric     CASE_MOVX_RM(SX, BQ)
2151*0fca6ea1SDimitry Andric     printSignExtend(MI, OutStreamer, 8, 64);
2152*0fca6ea1SDimitry Andric     break;
2153*0fca6ea1SDimitry Andric     CASE_MOVX_RM(SX, BW)
2154*0fca6ea1SDimitry Andric     printSignExtend(MI, OutStreamer, 8, 16);
2155*0fca6ea1SDimitry Andric     break;
2156*0fca6ea1SDimitry Andric     CASE_MOVX_RM(SX, DQ)
2157*0fca6ea1SDimitry Andric     printSignExtend(MI, OutStreamer, 32, 64);
2158*0fca6ea1SDimitry Andric     break;
2159*0fca6ea1SDimitry Andric     CASE_MOVX_RM(SX, WD)
2160*0fca6ea1SDimitry Andric     printSignExtend(MI, OutStreamer, 16, 32);
2161*0fca6ea1SDimitry Andric     break;
2162*0fca6ea1SDimitry Andric     CASE_MOVX_RM(SX, WQ)
2163*0fca6ea1SDimitry Andric     printSignExtend(MI, OutStreamer, 16, 64);
2164*0fca6ea1SDimitry Andric     break;
2165*0fca6ea1SDimitry Andric 
2166*0fca6ea1SDimitry Andric     CASE_MOVX_RM(ZX, BD)
2167*0fca6ea1SDimitry Andric     printZeroExtend(MI, OutStreamer, 8, 32);
2168*0fca6ea1SDimitry Andric     break;
2169*0fca6ea1SDimitry Andric     CASE_MOVX_RM(ZX, BQ)
2170*0fca6ea1SDimitry Andric     printZeroExtend(MI, OutStreamer, 8, 64);
2171*0fca6ea1SDimitry Andric     break;
2172*0fca6ea1SDimitry Andric     CASE_MOVX_RM(ZX, BW)
2173*0fca6ea1SDimitry Andric     printZeroExtend(MI, OutStreamer, 8, 16);
2174*0fca6ea1SDimitry Andric     break;
2175*0fca6ea1SDimitry Andric     CASE_MOVX_RM(ZX, DQ)
2176*0fca6ea1SDimitry Andric     printZeroExtend(MI, OutStreamer, 32, 64);
2177*0fca6ea1SDimitry Andric     break;
2178*0fca6ea1SDimitry Andric     CASE_MOVX_RM(ZX, WD)
2179*0fca6ea1SDimitry Andric     printZeroExtend(MI, OutStreamer, 16, 32);
2180*0fca6ea1SDimitry Andric     break;
2181*0fca6ea1SDimitry Andric     CASE_MOVX_RM(ZX, WQ)
2182*0fca6ea1SDimitry Andric     printZeroExtend(MI, OutStreamer, 16, 64);
2183*0fca6ea1SDimitry Andric     break;
21845ffd83dbSDimitry Andric   }
21855ffd83dbSDimitry Andric }
21865ffd83dbSDimitry Andric 
21875ffd83dbSDimitry Andric void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2188753f127fSDimitry Andric   // FIXME: Enable feature predicate checks once all the test pass.
2189753f127fSDimitry Andric   // X86_MC::verifyInstructionPredicates(MI->getOpcode(),
2190753f127fSDimitry Andric   //                                     Subtarget->getFeatureBits());
2191753f127fSDimitry Andric 
21920b57cec5SDimitry Andric   X86MCInstLower MCInstLowering(*MF, *this);
21930b57cec5SDimitry Andric   const X86RegisterInfo *RI =
21940b57cec5SDimitry Andric       MF->getSubtarget<X86Subtarget>().getRegisterInfo();
21950b57cec5SDimitry Andric 
21960eae32dcSDimitry Andric   if (MI->getOpcode() == X86::OR64rm) {
21970eae32dcSDimitry Andric     for (auto &Opd : MI->operands()) {
21980eae32dcSDimitry Andric       if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
21990eae32dcSDimitry Andric                                 "swift_async_extendedFramePointerFlags") {
22000eae32dcSDimitry Andric         ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
22010eae32dcSDimitry Andric       }
22020eae32dcSDimitry Andric     }
22030eae32dcSDimitry Andric   }
22040eae32dcSDimitry Andric 
22057a6dacacSDimitry Andric   // Add comments for values loaded from constant pool.
22067a6dacacSDimitry Andric   if (OutStreamer->isVerboseAsm())
22077a6dacacSDimitry Andric     addConstantComments(MI, *OutStreamer);
22087a6dacacSDimitry Andric 
22091db9f3b2SDimitry Andric   // Add a comment about EVEX compression
22100b57cec5SDimitry Andric   if (TM.Options.MCOptions.ShowMCEncoding) {
22111db9f3b2SDimitry Andric     if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY)
22121db9f3b2SDimitry Andric       OutStreamer->AddComment("EVEX TO LEGACY Compression ", false);
22131db9f3b2SDimitry Andric     else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
22140b57cec5SDimitry Andric       OutStreamer->AddComment("EVEX TO VEX Compression ", false);
22157a6dacacSDimitry Andric     else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX)
22167a6dacacSDimitry Andric       OutStreamer->AddComment("EVEX TO EVEX Compression ", false);
22170b57cec5SDimitry Andric   }
22180b57cec5SDimitry Andric 
22190b57cec5SDimitry Andric   switch (MI->getOpcode()) {
22200b57cec5SDimitry Andric   case TargetOpcode::DBG_VALUE:
22210b57cec5SDimitry Andric     llvm_unreachable("Should be handled target independently");
22220b57cec5SDimitry Andric 
22230b57cec5SDimitry Andric   case X86::EH_RETURN:
22240b57cec5SDimitry Andric   case X86::EH_RETURN64: {
22250b57cec5SDimitry Andric     // Lower these as normal, but add some comments.
22268bcb0991SDimitry Andric     Register Reg = MI->getOperand(0).getReg();
22270b57cec5SDimitry Andric     OutStreamer->AddComment(StringRef("eh_return, addr: %") +
22280b57cec5SDimitry Andric                             X86ATTInstPrinter::getRegisterName(Reg));
22290b57cec5SDimitry Andric     break;
22300b57cec5SDimitry Andric   }
22310b57cec5SDimitry Andric   case X86::CLEANUPRET: {
22320b57cec5SDimitry Andric     // Lower these as normal, but add some comments.
22330b57cec5SDimitry Andric     OutStreamer->AddComment("CLEANUPRET");
22340b57cec5SDimitry Andric     break;
22350b57cec5SDimitry Andric   }
22360b57cec5SDimitry Andric 
22370b57cec5SDimitry Andric   case X86::CATCHRET: {
22380b57cec5SDimitry Andric     // Lower these as normal, but add some comments.
22390b57cec5SDimitry Andric     OutStreamer->AddComment("CATCHRET");
22400b57cec5SDimitry Andric     break;
22410b57cec5SDimitry Andric   }
22420b57cec5SDimitry Andric 
224313138422SDimitry Andric   case X86::ENDBR32:
224413138422SDimitry Andric   case X86::ENDBR64: {
224513138422SDimitry Andric     // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
224613138422SDimitry Andric     // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
224713138422SDimitry Andric     // non-empty. If MI is the initial ENDBR, place the
224813138422SDimitry Andric     // __patchable_function_entries label after ENDBR.
224913138422SDimitry Andric     if (CurrentPatchableFunctionEntrySym &&
225013138422SDimitry Andric         CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
225113138422SDimitry Andric         MI == &MF->front().front()) {
225213138422SDimitry Andric       MCInst Inst;
225313138422SDimitry Andric       MCInstLowering.Lower(MI, Inst);
225413138422SDimitry Andric       EmitAndCountInstruction(Inst);
225513138422SDimitry Andric       CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
22565ffd83dbSDimitry Andric       OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
225713138422SDimitry Andric       return;
225813138422SDimitry Andric     }
225913138422SDimitry Andric     break;
226013138422SDimitry Andric   }
226113138422SDimitry Andric 
2262bdd1243dSDimitry Andric   case X86::TAILJMPd64:
2263bdd1243dSDimitry Andric     if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))
2264bdd1243dSDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX));
2265bdd1243dSDimitry Andric     [[fallthrough]];
22660b57cec5SDimitry Andric   case X86::TAILJMPr:
22670b57cec5SDimitry Andric   case X86::TAILJMPm:
22680b57cec5SDimitry Andric   case X86::TAILJMPd:
22690b57cec5SDimitry Andric   case X86::TAILJMPd_CC:
22700b57cec5SDimitry Andric   case X86::TAILJMPr64:
22710b57cec5SDimitry Andric   case X86::TAILJMPm64:
22720b57cec5SDimitry Andric   case X86::TAILJMPd64_CC:
22730b57cec5SDimitry Andric   case X86::TAILJMPr64_REX:
22740b57cec5SDimitry Andric   case X86::TAILJMPm64_REX:
22750b57cec5SDimitry Andric     // Lower these as normal, but add some comments.
22760b57cec5SDimitry Andric     OutStreamer->AddComment("TAILCALL");
22770b57cec5SDimitry Andric     break;
22780b57cec5SDimitry Andric 
22790b57cec5SDimitry Andric   case X86::TLS_addr32:
22800b57cec5SDimitry Andric   case X86::TLS_addr64:
2281e8d8bef9SDimitry Andric   case X86::TLS_addrX32:
22820b57cec5SDimitry Andric   case X86::TLS_base_addr32:
22830b57cec5SDimitry Andric   case X86::TLS_base_addr64:
2284e8d8bef9SDimitry Andric   case X86::TLS_base_addrX32:
2285*0fca6ea1SDimitry Andric   case X86::TLS_desc32:
2286*0fca6ea1SDimitry Andric   case X86::TLS_desc64:
22870b57cec5SDimitry Andric     return LowerTlsAddr(MCInstLowering, *MI);
22880b57cec5SDimitry Andric 
22890b57cec5SDimitry Andric   case X86::MOVPC32r: {
22900b57cec5SDimitry Andric     // This is a pseudo op for a two instruction sequence with a label, which
22910b57cec5SDimitry Andric     // looks like:
22920b57cec5SDimitry Andric     //     call "L1$pb"
22930b57cec5SDimitry Andric     // "L1$pb":
22940b57cec5SDimitry Andric     //     popl %esi
22950b57cec5SDimitry Andric 
22960b57cec5SDimitry Andric     // Emit the call.
22970b57cec5SDimitry Andric     MCSymbol *PICBase = MF->getPICBaseSymbol();
22980b57cec5SDimitry Andric     // FIXME: We would like an efficient form for this, so we don't have to do a
22990b57cec5SDimitry Andric     // lot of extra uniquing.
23000b57cec5SDimitry Andric     EmitAndCountInstruction(
23010b57cec5SDimitry Andric         MCInstBuilder(X86::CALLpcrel32)
23020b57cec5SDimitry Andric             .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
23030b57cec5SDimitry Andric 
23040b57cec5SDimitry Andric     const X86FrameLowering *FrameLowering =
23050b57cec5SDimitry Andric         MF->getSubtarget<X86Subtarget>().getFrameLowering();
23060b57cec5SDimitry Andric     bool hasFP = FrameLowering->hasFP(*MF);
23070b57cec5SDimitry Andric 
23080b57cec5SDimitry Andric     // TODO: This is needed only if we require precise CFA.
23090b57cec5SDimitry Andric     bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
23100b57cec5SDimitry Andric                                !OutStreamer->getDwarfFrameInfos().back().End;
23110b57cec5SDimitry Andric 
23120b57cec5SDimitry Andric     int stackGrowth = -RI->getSlotSize();
23130b57cec5SDimitry Andric 
23140b57cec5SDimitry Andric     if (HasActiveDwarfFrame && !hasFP) {
23155ffd83dbSDimitry Andric       OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);
231606c3fb27SDimitry Andric       MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
23170b57cec5SDimitry Andric     }
23180b57cec5SDimitry Andric 
23190b57cec5SDimitry Andric     // Emit the label.
23205ffd83dbSDimitry Andric     OutStreamer->emitLabel(PICBase);
23210b57cec5SDimitry Andric 
23220b57cec5SDimitry Andric     // popl $reg
23230b57cec5SDimitry Andric     EmitAndCountInstruction(
23240b57cec5SDimitry Andric         MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));
23250b57cec5SDimitry Andric 
23260b57cec5SDimitry Andric     if (HasActiveDwarfFrame && !hasFP) {
23275ffd83dbSDimitry Andric       OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);
23280b57cec5SDimitry Andric     }
23290b57cec5SDimitry Andric     return;
23300b57cec5SDimitry Andric   }
23310b57cec5SDimitry Andric 
23320b57cec5SDimitry Andric   case X86::ADD32ri: {
23330b57cec5SDimitry Andric     // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
23340b57cec5SDimitry Andric     if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
23350b57cec5SDimitry Andric       break;
23360b57cec5SDimitry Andric 
23370b57cec5SDimitry Andric     // Okay, we have something like:
23380b57cec5SDimitry Andric     //  EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
23390b57cec5SDimitry Andric 
23400b57cec5SDimitry Andric     // For this, we want to print something like:
23410b57cec5SDimitry Andric     //   MYGLOBAL + (. - PICBASE)
23420b57cec5SDimitry Andric     // However, we can't generate a ".", so just emit a new label here and refer
23430b57cec5SDimitry Andric     // to it.
23440b57cec5SDimitry Andric     MCSymbol *DotSym = OutContext.createTempSymbol();
23455ffd83dbSDimitry Andric     OutStreamer->emitLabel(DotSym);
23460b57cec5SDimitry Andric 
23470b57cec5SDimitry Andric     // Now that we have emitted the label, lower the complex operand expression.
23480b57cec5SDimitry Andric     MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
23490b57cec5SDimitry Andric 
23500b57cec5SDimitry Andric     const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
23510b57cec5SDimitry Andric     const MCExpr *PICBase =
23520b57cec5SDimitry Andric         MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);
23530b57cec5SDimitry Andric     DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);
23540b57cec5SDimitry Andric 
23550b57cec5SDimitry Andric     DotExpr = MCBinaryExpr::createAdd(
23560b57cec5SDimitry Andric         MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext);
23570b57cec5SDimitry Andric 
23580b57cec5SDimitry Andric     EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)
23590b57cec5SDimitry Andric                                 .addReg(MI->getOperand(0).getReg())
23600b57cec5SDimitry Andric                                 .addReg(MI->getOperand(1).getReg())
23610b57cec5SDimitry Andric                                 .addExpr(DotExpr));
23620b57cec5SDimitry Andric     return;
23630b57cec5SDimitry Andric   }
23640b57cec5SDimitry Andric   case TargetOpcode::STATEPOINT:
23650b57cec5SDimitry Andric     return LowerSTATEPOINT(*MI, MCInstLowering);
23660b57cec5SDimitry Andric 
23670b57cec5SDimitry Andric   case TargetOpcode::FAULTING_OP:
23680b57cec5SDimitry Andric     return LowerFAULTING_OP(*MI, MCInstLowering);
23690b57cec5SDimitry Andric 
23700b57cec5SDimitry Andric   case TargetOpcode::FENTRY_CALL:
23710b57cec5SDimitry Andric     return LowerFENTRY_CALL(*MI, MCInstLowering);
23720b57cec5SDimitry Andric 
23730b57cec5SDimitry Andric   case TargetOpcode::PATCHABLE_OP:
23740b57cec5SDimitry Andric     return LowerPATCHABLE_OP(*MI, MCInstLowering);
23750b57cec5SDimitry Andric 
23760b57cec5SDimitry Andric   case TargetOpcode::STACKMAP:
23770b57cec5SDimitry Andric     return LowerSTACKMAP(*MI);
23780b57cec5SDimitry Andric 
23790b57cec5SDimitry Andric   case TargetOpcode::PATCHPOINT:
23800b57cec5SDimitry Andric     return LowerPATCHPOINT(*MI, MCInstLowering);
23810b57cec5SDimitry Andric 
23820b57cec5SDimitry Andric   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
23830b57cec5SDimitry Andric     return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);
23840b57cec5SDimitry Andric 
23850b57cec5SDimitry Andric   case TargetOpcode::PATCHABLE_RET:
23860b57cec5SDimitry Andric     return LowerPATCHABLE_RET(*MI, MCInstLowering);
23870b57cec5SDimitry Andric 
23880b57cec5SDimitry Andric   case TargetOpcode::PATCHABLE_TAIL_CALL:
23890b57cec5SDimitry Andric     return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);
23900b57cec5SDimitry Andric 
23910b57cec5SDimitry Andric   case TargetOpcode::PATCHABLE_EVENT_CALL:
23920b57cec5SDimitry Andric     return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering);
23930b57cec5SDimitry Andric 
23940b57cec5SDimitry Andric   case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
23950b57cec5SDimitry Andric     return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering);
23960b57cec5SDimitry Andric 
23970b57cec5SDimitry Andric   case X86::MORESTACK_RET:
23980b57cec5SDimitry Andric     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
23990b57cec5SDimitry Andric     return;
24000b57cec5SDimitry Andric 
2401bdd1243dSDimitry Andric   case X86::KCFI_CHECK:
2402bdd1243dSDimitry Andric     return LowerKCFI_CHECK(*MI);
2403bdd1243dSDimitry Andric 
2404349cc55cSDimitry Andric   case X86::ASAN_CHECK_MEMACCESS:
2405349cc55cSDimitry Andric     return LowerASAN_CHECK_MEMACCESS(*MI);
2406349cc55cSDimitry Andric 
24070b57cec5SDimitry Andric   case X86::MORESTACK_RET_RESTORE_R10:
24080b57cec5SDimitry Andric     // Return, then restore R10.
24090b57cec5SDimitry Andric     EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
24100b57cec5SDimitry Andric     EmitAndCountInstruction(
24110b57cec5SDimitry Andric         MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX));
24120b57cec5SDimitry Andric     return;
24130b57cec5SDimitry Andric 
24140b57cec5SDimitry Andric   case X86::SEH_PushReg:
24150b57cec5SDimitry Andric   case X86::SEH_SaveReg:
24160b57cec5SDimitry Andric   case X86::SEH_SaveXMM:
24170b57cec5SDimitry Andric   case X86::SEH_StackAlloc:
24180b57cec5SDimitry Andric   case X86::SEH_StackAlign:
24190b57cec5SDimitry Andric   case X86::SEH_SetFrame:
24200b57cec5SDimitry Andric   case X86::SEH_PushFrame:
24210b57cec5SDimitry Andric   case X86::SEH_EndPrologue:
24220b57cec5SDimitry Andric     EmitSEHInstruction(MI);
24230b57cec5SDimitry Andric     return;
24240b57cec5SDimitry Andric 
24250b57cec5SDimitry Andric   case X86::SEH_Epilogue: {
24260b57cec5SDimitry Andric     assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
24270b57cec5SDimitry Andric     MachineBasicBlock::const_iterator MBBI(MI);
24280b57cec5SDimitry Andric     // Check if preceded by a call and emit nop if so.
24290b57cec5SDimitry Andric     for (MBBI = PrevCrossBBInst(MBBI);
24300b57cec5SDimitry Andric          MBBI != MachineBasicBlock::const_iterator();
24310b57cec5SDimitry Andric          MBBI = PrevCrossBBInst(MBBI)) {
243206c3fb27SDimitry Andric       // Pseudo instructions that aren't a call are assumed to not emit any
243306c3fb27SDimitry Andric       // code. If they do, we worst case generate unnecessary noops after a
243406c3fb27SDimitry Andric       // call.
243506c3fb27SDimitry Andric       if (MBBI->isCall() || !MBBI->isPseudo()) {
24360b57cec5SDimitry Andric         if (MBBI->isCall())
24370b57cec5SDimitry Andric           EmitAndCountInstruction(MCInstBuilder(X86::NOOP));
24380b57cec5SDimitry Andric         break;
24390b57cec5SDimitry Andric       }
24400b57cec5SDimitry Andric     }
24410b57cec5SDimitry Andric     return;
24420b57cec5SDimitry Andric   }
2443e8d8bef9SDimitry Andric   case X86::UBSAN_UD1:
2444e8d8bef9SDimitry Andric     EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)
2445e8d8bef9SDimitry Andric                                 .addReg(X86::EAX)
2446e8d8bef9SDimitry Andric                                 .addReg(X86::EAX)
2447e8d8bef9SDimitry Andric                                 .addImm(1)
2448e8d8bef9SDimitry Andric                                 .addReg(X86::NoRegister)
2449e8d8bef9SDimitry Andric                                 .addImm(MI->getOperand(0).getImm())
2450e8d8bef9SDimitry Andric                                 .addReg(X86::NoRegister));
2451e8d8bef9SDimitry Andric     return;
2452bdd1243dSDimitry Andric   case X86::CALL64pcrel32:
2453bdd1243dSDimitry Andric     if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))
2454bdd1243dSDimitry Andric       EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX));
2455bdd1243dSDimitry Andric     break;
2456*0fca6ea1SDimitry Andric   case X86::JCC_1:
2457*0fca6ea1SDimitry Andric     // Two instruction prefixes (2EH for branch not-taken and 3EH for branch
2458*0fca6ea1SDimitry Andric     // taken) are used as branch hints. Here we add branch taken prefix for
2459*0fca6ea1SDimitry Andric     // jump instruction with higher probability than threshold.
2460*0fca6ea1SDimitry Andric     if (getSubtarget().hasBranchHint() && EnableBranchHint) {
2461*0fca6ea1SDimitry Andric       const MachineBranchProbabilityInfo *MBPI =
2462*0fca6ea1SDimitry Andric           &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
2463*0fca6ea1SDimitry Andric       MachineBasicBlock *DestBB = MI->getOperand(0).getMBB();
2464*0fca6ea1SDimitry Andric       BranchProbability EdgeProb =
2465*0fca6ea1SDimitry Andric           MBPI->getEdgeProbability(MI->getParent(), DestBB);
2466*0fca6ea1SDimitry Andric       BranchProbability Threshold(BranchHintProbabilityThreshold, 100);
2467*0fca6ea1SDimitry Andric       if (EdgeProb > Threshold)
2468*0fca6ea1SDimitry Andric         EmitAndCountInstruction(MCInstBuilder(X86::DS_PREFIX));
2469*0fca6ea1SDimitry Andric     }
2470*0fca6ea1SDimitry Andric     break;
24710b57cec5SDimitry Andric   }
24720b57cec5SDimitry Andric 
24730b57cec5SDimitry Andric   MCInst TmpInst;
24740b57cec5SDimitry Andric   MCInstLowering.Lower(MI, TmpInst);
24750b57cec5SDimitry Andric 
24760b57cec5SDimitry Andric   // Stackmap shadows cannot include branch targets, so we can count the bytes
24770b57cec5SDimitry Andric   // in a call towards the shadow, but must ensure that the no thread returns
24780b57cec5SDimitry Andric   // in to the stackmap shadow.  The only way to achieve this is if the call
24790b57cec5SDimitry Andric   // is at the end of the shadow.
24800b57cec5SDimitry Andric   if (MI->isCall()) {
24810b57cec5SDimitry Andric     // Count then size of the call towards the shadow
24820b57cec5SDimitry Andric     SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get());
24830b57cec5SDimitry Andric     // Then flush the shadow so that we fill with nops before the call, not
24840b57cec5SDimitry Andric     // after it.
24850b57cec5SDimitry Andric     SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
24860b57cec5SDimitry Andric     // Then emit the call
24875ffd83dbSDimitry Andric     OutStreamer->emitInstruction(TmpInst, getSubtargetInfo());
24880b57cec5SDimitry Andric     return;
24890b57cec5SDimitry Andric   }
24900b57cec5SDimitry Andric 
24910b57cec5SDimitry Andric   EmitAndCountInstruction(TmpInst);
24920b57cec5SDimitry Andric }
2493