10b57cec5SDimitry Andric //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file contains code to lower X86 MachineInstrs to their corresponding 100b57cec5SDimitry Andric // MCInst records. 110b57cec5SDimitry Andric // 120b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andric #include "MCTargetDesc/X86ATTInstPrinter.h" 150b57cec5SDimitry Andric #include "MCTargetDesc/X86BaseInfo.h" 1606c3fb27SDimitry Andric #include "MCTargetDesc/X86EncodingOptimization.h" 170b57cec5SDimitry Andric #include "MCTargetDesc/X86InstComments.h" 185ffd83dbSDimitry Andric #include "MCTargetDesc/X86ShuffleDecode.h" 190b57cec5SDimitry Andric #include "MCTargetDesc/X86TargetStreamer.h" 200b57cec5SDimitry Andric #include "X86AsmPrinter.h" 2106c3fb27SDimitry Andric #include "X86MachineFunctionInfo.h" 220b57cec5SDimitry Andric #include "X86RegisterInfo.h" 230b57cec5SDimitry Andric #include "X86ShuffleDecodeConstantPool.h" 245ffd83dbSDimitry Andric #include "X86Subtarget.h" 25*0fca6ea1SDimitry Andric #include "llvm/ADT/STLExtras.h" 260b57cec5SDimitry Andric #include "llvm/ADT/SmallString.h" 2706c3fb27SDimitry Andric #include "llvm/ADT/StringExtras.h" 28*0fca6ea1SDimitry Andric #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" 290b57cec5SDimitry Andric #include "llvm/CodeGen/MachineConstantPool.h" 300b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h" 310b57cec5SDimitry Andric #include "llvm/CodeGen/MachineModuleInfoImpls.h" 320b57cec5SDimitry Andric #include "llvm/CodeGen/MachineOperand.h" 330b57cec5SDimitry Andric #include "llvm/CodeGen/StackMaps.h" 340b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h" 350b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h" 360b57cec5SDimitry Andric #include "llvm/IR/Mangler.h" 370b57cec5SDimitry Andric #include "llvm/MC/MCAsmInfo.h" 380b57cec5SDimitry Andric #include "llvm/MC/MCCodeEmitter.h" 390b57cec5SDimitry Andric #include "llvm/MC/MCContext.h" 400b57cec5SDimitry Andric #include "llvm/MC/MCExpr.h" 410b57cec5SDimitry Andric #include "llvm/MC/MCFixup.h" 420b57cec5SDimitry Andric #include "llvm/MC/MCInst.h" 430b57cec5SDimitry Andric #include "llvm/MC/MCInstBuilder.h" 440b57cec5SDimitry Andric #include "llvm/MC/MCSection.h" 450b57cec5SDimitry Andric #include "llvm/MC/MCSectionELF.h" 460b57cec5SDimitry Andric #include "llvm/MC/MCStreamer.h" 470b57cec5SDimitry Andric #include "llvm/MC/MCSymbol.h" 480b57cec5SDimitry Andric #include "llvm/MC/MCSymbolELF.h" 49349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h" 500b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h" 515ffd83dbSDimitry Andric #include "llvm/Target/TargetMachine.h" 52349cc55cSDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" 53349cc55cSDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h" 540eae32dcSDimitry Andric #include <string> 550b57cec5SDimitry Andric 560b57cec5SDimitry Andric using namespace llvm; 570b57cec5SDimitry Andric 58*0fca6ea1SDimitry Andric static cl::opt<bool> EnableBranchHint("enable-branch-hint", 59*0fca6ea1SDimitry Andric cl::desc("Enable branch hint."), 60*0fca6ea1SDimitry Andric cl::init(false), cl::Hidden); 61*0fca6ea1SDimitry Andric static cl::opt<unsigned> BranchHintProbabilityThreshold( 62*0fca6ea1SDimitry Andric "branch-hint-probability-threshold", 63*0fca6ea1SDimitry Andric cl::desc("The probability threshold of enabling branch hint."), 64*0fca6ea1SDimitry Andric cl::init(50), cl::Hidden); 65*0fca6ea1SDimitry Andric 660b57cec5SDimitry Andric namespace { 670b57cec5SDimitry Andric 680b57cec5SDimitry Andric /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst. 690b57cec5SDimitry Andric class X86MCInstLower { 700b57cec5SDimitry Andric MCContext &Ctx; 710b57cec5SDimitry Andric const MachineFunction &MF; 720b57cec5SDimitry Andric const TargetMachine &TM; 730b57cec5SDimitry Andric const MCAsmInfo &MAI; 740b57cec5SDimitry Andric X86AsmPrinter &AsmPrinter; 750b57cec5SDimitry Andric 760b57cec5SDimitry Andric public: 770b57cec5SDimitry Andric X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter); 780b57cec5SDimitry Andric 79*0fca6ea1SDimitry Andric MCOperand LowerMachineOperand(const MachineInstr *MI, 800b57cec5SDimitry Andric const MachineOperand &MO) const; 810b57cec5SDimitry Andric void Lower(const MachineInstr *MI, MCInst &OutMI) const; 820b57cec5SDimitry Andric 830b57cec5SDimitry Andric MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const; 840b57cec5SDimitry Andric MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const; 850b57cec5SDimitry Andric 860b57cec5SDimitry Andric private: 870b57cec5SDimitry Andric MachineModuleInfoMachO &getMachOMMI() const; 880b57cec5SDimitry Andric }; 890b57cec5SDimitry Andric 900b57cec5SDimitry Andric } // end anonymous namespace 910b57cec5SDimitry Andric 925ffd83dbSDimitry Andric /// A RAII helper which defines a region of instructions which can't have 935ffd83dbSDimitry Andric /// padding added between them for correctness. 945ffd83dbSDimitry Andric struct NoAutoPaddingScope { 955ffd83dbSDimitry Andric MCStreamer &OS; 965ffd83dbSDimitry Andric const bool OldAllowAutoPadding; 975ffd83dbSDimitry Andric NoAutoPaddingScope(MCStreamer &OS) 985ffd83dbSDimitry Andric : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) { 995ffd83dbSDimitry Andric changeAndComment(false); 1005ffd83dbSDimitry Andric } 1015ffd83dbSDimitry Andric ~NoAutoPaddingScope() { changeAndComment(OldAllowAutoPadding); } 1025ffd83dbSDimitry Andric void changeAndComment(bool b) { 1035ffd83dbSDimitry Andric if (b == OS.getAllowAutoPadding()) 1045ffd83dbSDimitry Andric return; 1055ffd83dbSDimitry Andric OS.setAllowAutoPadding(b); 1065ffd83dbSDimitry Andric if (b) 1075ffd83dbSDimitry Andric OS.emitRawComment("autopadding"); 1085ffd83dbSDimitry Andric else 1095ffd83dbSDimitry Andric OS.emitRawComment("noautopadding"); 1105ffd83dbSDimitry Andric } 1115ffd83dbSDimitry Andric }; 1125ffd83dbSDimitry Andric 1130b57cec5SDimitry Andric // Emit a minimal sequence of nops spanning NumBytes bytes. 1145ffd83dbSDimitry Andric static void emitX86Nops(MCStreamer &OS, unsigned NumBytes, 1155ffd83dbSDimitry Andric const X86Subtarget *Subtarget); 1160b57cec5SDimitry Andric 1170b57cec5SDimitry Andric void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst, 1180b57cec5SDimitry Andric const MCSubtargetInfo &STI, 1190b57cec5SDimitry Andric MCCodeEmitter *CodeEmitter) { 1200b57cec5SDimitry Andric if (InShadow) { 1210b57cec5SDimitry Andric SmallString<256> Code; 1220b57cec5SDimitry Andric SmallVector<MCFixup, 4> Fixups; 12306c3fb27SDimitry Andric CodeEmitter->encodeInstruction(Inst, Code, Fixups, STI); 1240b57cec5SDimitry Andric CurrentShadowSize += Code.size(); 1250b57cec5SDimitry Andric if (CurrentShadowSize >= RequiredShadowSize) 1260b57cec5SDimitry Andric InShadow = false; // The shadow is big enough. Stop counting. 1270b57cec5SDimitry Andric } 1280b57cec5SDimitry Andric } 1290b57cec5SDimitry Andric 1300b57cec5SDimitry Andric void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding( 1310b57cec5SDimitry Andric MCStreamer &OutStreamer, const MCSubtargetInfo &STI) { 1320b57cec5SDimitry Andric if (InShadow && CurrentShadowSize < RequiredShadowSize) { 1330b57cec5SDimitry Andric InShadow = false; 1345ffd83dbSDimitry Andric emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize, 1355ffd83dbSDimitry Andric &MF->getSubtarget<X86Subtarget>()); 1360b57cec5SDimitry Andric } 1370b57cec5SDimitry Andric } 1380b57cec5SDimitry Andric 1390b57cec5SDimitry Andric void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) { 1405ffd83dbSDimitry Andric OutStreamer->emitInstruction(Inst, getSubtargetInfo()); 1410b57cec5SDimitry Andric SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get()); 1420b57cec5SDimitry Andric } 1430b57cec5SDimitry Andric 1440b57cec5SDimitry Andric X86MCInstLower::X86MCInstLower(const MachineFunction &mf, 1450b57cec5SDimitry Andric X86AsmPrinter &asmprinter) 1460b57cec5SDimitry Andric : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()), 1470b57cec5SDimitry Andric AsmPrinter(asmprinter) {} 1480b57cec5SDimitry Andric 1490b57cec5SDimitry Andric MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const { 150*0fca6ea1SDimitry Andric return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>(); 1510b57cec5SDimitry Andric } 1520b57cec5SDimitry Andric 1530b57cec5SDimitry Andric /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol 1540b57cec5SDimitry Andric /// operand to an MCSymbol. 1550b57cec5SDimitry Andric MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const { 1565ffd83dbSDimitry Andric const Triple &TT = TM.getTargetTriple(); 1575ffd83dbSDimitry Andric if (MO.isGlobal() && TT.isOSBinFormatELF()) 1585ffd83dbSDimitry Andric return AsmPrinter.getSymbolPreferLocal(*MO.getGlobal()); 1595ffd83dbSDimitry Andric 1600b57cec5SDimitry Andric const DataLayout &DL = MF.getDataLayout(); 1610b57cec5SDimitry Andric assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && 1620b57cec5SDimitry Andric "Isn't a symbol reference"); 1630b57cec5SDimitry Andric 1640b57cec5SDimitry Andric MCSymbol *Sym = nullptr; 1650b57cec5SDimitry Andric SmallString<128> Name; 1660b57cec5SDimitry Andric StringRef Suffix; 1670b57cec5SDimitry Andric 1680b57cec5SDimitry Andric switch (MO.getTargetFlags()) { 1690b57cec5SDimitry Andric case X86II::MO_DLLIMPORT: 1700b57cec5SDimitry Andric // Handle dllimport linkage. 1710b57cec5SDimitry Andric Name += "__imp_"; 1720b57cec5SDimitry Andric break; 1730b57cec5SDimitry Andric case X86II::MO_COFFSTUB: 1740b57cec5SDimitry Andric Name += ".refptr."; 1750b57cec5SDimitry Andric break; 1760b57cec5SDimitry Andric case X86II::MO_DARWIN_NONLAZY: 1770b57cec5SDimitry Andric case X86II::MO_DARWIN_NONLAZY_PIC_BASE: 1780b57cec5SDimitry Andric Suffix = "$non_lazy_ptr"; 1790b57cec5SDimitry Andric break; 1800b57cec5SDimitry Andric } 1810b57cec5SDimitry Andric 1820b57cec5SDimitry Andric if (!Suffix.empty()) 1830b57cec5SDimitry Andric Name += DL.getPrivateGlobalPrefix(); 1840b57cec5SDimitry Andric 1850b57cec5SDimitry Andric if (MO.isGlobal()) { 1860b57cec5SDimitry Andric const GlobalValue *GV = MO.getGlobal(); 1870b57cec5SDimitry Andric AsmPrinter.getNameWithPrefix(Name, GV); 1880b57cec5SDimitry Andric } else if (MO.isSymbol()) { 1890b57cec5SDimitry Andric Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL); 1900b57cec5SDimitry Andric } else if (MO.isMBB()) { 1910b57cec5SDimitry Andric assert(Suffix.empty()); 1920b57cec5SDimitry Andric Sym = MO.getMBB()->getSymbol(); 1930b57cec5SDimitry Andric } 1940b57cec5SDimitry Andric 1950b57cec5SDimitry Andric Name += Suffix; 1960b57cec5SDimitry Andric if (!Sym) 1970b57cec5SDimitry Andric Sym = Ctx.getOrCreateSymbol(Name); 1980b57cec5SDimitry Andric 1990b57cec5SDimitry Andric // If the target flags on the operand changes the name of the symbol, do that 2000b57cec5SDimitry Andric // before we return the symbol. 2010b57cec5SDimitry Andric switch (MO.getTargetFlags()) { 2020b57cec5SDimitry Andric default: 2030b57cec5SDimitry Andric break; 2040b57cec5SDimitry Andric case X86II::MO_COFFSTUB: { 2050b57cec5SDimitry Andric MachineModuleInfoCOFF &MMICOFF = 206*0fca6ea1SDimitry Andric AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>(); 2070b57cec5SDimitry Andric MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym); 2080b57cec5SDimitry Andric if (!StubSym.getPointer()) { 2090b57cec5SDimitry Andric assert(MO.isGlobal() && "Extern symbol not handled yet"); 2100b57cec5SDimitry Andric StubSym = MachineModuleInfoImpl::StubValueTy( 2110b57cec5SDimitry Andric AsmPrinter.getSymbol(MO.getGlobal()), true); 2120b57cec5SDimitry Andric } 2130b57cec5SDimitry Andric break; 2140b57cec5SDimitry Andric } 2150b57cec5SDimitry Andric case X86II::MO_DARWIN_NONLAZY: 2160b57cec5SDimitry Andric case X86II::MO_DARWIN_NONLAZY_PIC_BASE: { 2170b57cec5SDimitry Andric MachineModuleInfoImpl::StubValueTy &StubSym = 2180b57cec5SDimitry Andric getMachOMMI().getGVStubEntry(Sym); 2190b57cec5SDimitry Andric if (!StubSym.getPointer()) { 2200b57cec5SDimitry Andric assert(MO.isGlobal() && "Extern symbol not handled yet"); 2210b57cec5SDimitry Andric StubSym = MachineModuleInfoImpl::StubValueTy( 2220b57cec5SDimitry Andric AsmPrinter.getSymbol(MO.getGlobal()), 2230b57cec5SDimitry Andric !MO.getGlobal()->hasInternalLinkage()); 2240b57cec5SDimitry Andric } 2250b57cec5SDimitry Andric break; 2260b57cec5SDimitry Andric } 2270b57cec5SDimitry Andric } 2280b57cec5SDimitry Andric 2290b57cec5SDimitry Andric return Sym; 2300b57cec5SDimitry Andric } 2310b57cec5SDimitry Andric 2320b57cec5SDimitry Andric MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO, 2330b57cec5SDimitry Andric MCSymbol *Sym) const { 2340b57cec5SDimitry Andric // FIXME: We would like an efficient form for this, so we don't have to do a 2350b57cec5SDimitry Andric // lot of extra uniquing. 2360b57cec5SDimitry Andric const MCExpr *Expr = nullptr; 2370b57cec5SDimitry Andric MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None; 2380b57cec5SDimitry Andric 2390b57cec5SDimitry Andric switch (MO.getTargetFlags()) { 2400b57cec5SDimitry Andric default: 2410b57cec5SDimitry Andric llvm_unreachable("Unknown target flag on GV operand"); 2420b57cec5SDimitry Andric case X86II::MO_NO_FLAG: // No flag. 2430b57cec5SDimitry Andric // These affect the name of the symbol, not any suffix. 2440b57cec5SDimitry Andric case X86II::MO_DARWIN_NONLAZY: 2450b57cec5SDimitry Andric case X86II::MO_DLLIMPORT: 2460b57cec5SDimitry Andric case X86II::MO_COFFSTUB: 2470b57cec5SDimitry Andric break; 2480b57cec5SDimitry Andric 2490b57cec5SDimitry Andric case X86II::MO_TLVP: 2500b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_TLVP; 2510b57cec5SDimitry Andric break; 2520b57cec5SDimitry Andric case X86II::MO_TLVP_PIC_BASE: 2530b57cec5SDimitry Andric Expr = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_TLVP, Ctx); 2540b57cec5SDimitry Andric // Subtract the pic base. 2550b57cec5SDimitry Andric Expr = MCBinaryExpr::createSub( 2560b57cec5SDimitry Andric Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx); 2570b57cec5SDimitry Andric break; 2580b57cec5SDimitry Andric case X86II::MO_SECREL: 2590b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_SECREL; 2600b57cec5SDimitry Andric break; 2610b57cec5SDimitry Andric case X86II::MO_TLSGD: 2620b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_TLSGD; 2630b57cec5SDimitry Andric break; 2640b57cec5SDimitry Andric case X86II::MO_TLSLD: 2650b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_TLSLD; 2660b57cec5SDimitry Andric break; 2670b57cec5SDimitry Andric case X86II::MO_TLSLDM: 2680b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_TLSLDM; 2690b57cec5SDimitry Andric break; 2700b57cec5SDimitry Andric case X86II::MO_GOTTPOFF: 2710b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_GOTTPOFF; 2720b57cec5SDimitry Andric break; 2730b57cec5SDimitry Andric case X86II::MO_INDNTPOFF: 2740b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_INDNTPOFF; 2750b57cec5SDimitry Andric break; 2760b57cec5SDimitry Andric case X86II::MO_TPOFF: 2770b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_TPOFF; 2780b57cec5SDimitry Andric break; 2790b57cec5SDimitry Andric case X86II::MO_DTPOFF: 2800b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_DTPOFF; 2810b57cec5SDimitry Andric break; 2820b57cec5SDimitry Andric case X86II::MO_NTPOFF: 2830b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_NTPOFF; 2840b57cec5SDimitry Andric break; 2850b57cec5SDimitry Andric case X86II::MO_GOTNTPOFF: 2860b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_GOTNTPOFF; 2870b57cec5SDimitry Andric break; 2880b57cec5SDimitry Andric case X86II::MO_GOTPCREL: 2890b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_GOTPCREL; 2900b57cec5SDimitry Andric break; 291349cc55cSDimitry Andric case X86II::MO_GOTPCREL_NORELAX: 292349cc55cSDimitry Andric RefKind = MCSymbolRefExpr::VK_GOTPCREL_NORELAX; 293349cc55cSDimitry Andric break; 2940b57cec5SDimitry Andric case X86II::MO_GOT: 2950b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_GOT; 2960b57cec5SDimitry Andric break; 2970b57cec5SDimitry Andric case X86II::MO_GOTOFF: 2980b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_GOTOFF; 2990b57cec5SDimitry Andric break; 3000b57cec5SDimitry Andric case X86II::MO_PLT: 3010b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_PLT; 3020b57cec5SDimitry Andric break; 3030b57cec5SDimitry Andric case X86II::MO_ABS8: 3040b57cec5SDimitry Andric RefKind = MCSymbolRefExpr::VK_X86_ABS8; 3050b57cec5SDimitry Andric break; 3060b57cec5SDimitry Andric case X86II::MO_PIC_BASE_OFFSET: 3070b57cec5SDimitry Andric case X86II::MO_DARWIN_NONLAZY_PIC_BASE: 3080b57cec5SDimitry Andric Expr = MCSymbolRefExpr::create(Sym, Ctx); 3090b57cec5SDimitry Andric // Subtract the pic base. 3100b57cec5SDimitry Andric Expr = MCBinaryExpr::createSub( 3110b57cec5SDimitry Andric Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx); 3120b57cec5SDimitry Andric if (MO.isJTI()) { 3130b57cec5SDimitry Andric assert(MAI.doesSetDirectiveSuppressReloc()); 3140b57cec5SDimitry Andric // If .set directive is supported, use it to reduce the number of 3150b57cec5SDimitry Andric // relocations the assembler will generate for differences between 3160b57cec5SDimitry Andric // local labels. This is only safe when the symbols are in the same 3170b57cec5SDimitry Andric // section so we are restricting it to jumptable references. 3180b57cec5SDimitry Andric MCSymbol *Label = Ctx.createTempSymbol(); 3195ffd83dbSDimitry Andric AsmPrinter.OutStreamer->emitAssignment(Label, Expr); 3200b57cec5SDimitry Andric Expr = MCSymbolRefExpr::create(Label, Ctx); 3210b57cec5SDimitry Andric } 3220b57cec5SDimitry Andric break; 3230b57cec5SDimitry Andric } 3240b57cec5SDimitry Andric 3250b57cec5SDimitry Andric if (!Expr) 3260b57cec5SDimitry Andric Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx); 3270b57cec5SDimitry Andric 3280b57cec5SDimitry Andric if (!MO.isJTI() && !MO.isMBB() && MO.getOffset()) 3290b57cec5SDimitry Andric Expr = MCBinaryExpr::createAdd( 3300b57cec5SDimitry Andric Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx); 3310b57cec5SDimitry Andric return MCOperand::createExpr(Expr); 3320b57cec5SDimitry Andric } 3330b57cec5SDimitry Andric 3340b57cec5SDimitry Andric static unsigned getRetOpcode(const X86Subtarget &Subtarget) { 335349cc55cSDimitry Andric return Subtarget.is64Bit() ? X86::RET64 : X86::RET32; 3360b57cec5SDimitry Andric } 3370b57cec5SDimitry Andric 338*0fca6ea1SDimitry Andric MCOperand X86MCInstLower::LowerMachineOperand(const MachineInstr *MI, 3390b57cec5SDimitry Andric const MachineOperand &MO) const { 3400b57cec5SDimitry Andric switch (MO.getType()) { 3410b57cec5SDimitry Andric default: 3420b57cec5SDimitry Andric MI->print(errs()); 3430b57cec5SDimitry Andric llvm_unreachable("unknown operand type"); 3440b57cec5SDimitry Andric case MachineOperand::MO_Register: 3450b57cec5SDimitry Andric // Ignore all implicit register operands. 3460b57cec5SDimitry Andric if (MO.isImplicit()) 347*0fca6ea1SDimitry Andric return MCOperand(); 3480b57cec5SDimitry Andric return MCOperand::createReg(MO.getReg()); 3490b57cec5SDimitry Andric case MachineOperand::MO_Immediate: 3500b57cec5SDimitry Andric return MCOperand::createImm(MO.getImm()); 3510b57cec5SDimitry Andric case MachineOperand::MO_MachineBasicBlock: 3520b57cec5SDimitry Andric case MachineOperand::MO_GlobalAddress: 3530b57cec5SDimitry Andric case MachineOperand::MO_ExternalSymbol: 3540b57cec5SDimitry Andric return LowerSymbolOperand(MO, GetSymbolFromOperand(MO)); 3550b57cec5SDimitry Andric case MachineOperand::MO_MCSymbol: 3560b57cec5SDimitry Andric return LowerSymbolOperand(MO, MO.getMCSymbol()); 3570b57cec5SDimitry Andric case MachineOperand::MO_JumpTableIndex: 3580b57cec5SDimitry Andric return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex())); 3590b57cec5SDimitry Andric case MachineOperand::MO_ConstantPoolIndex: 3600b57cec5SDimitry Andric return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex())); 3610b57cec5SDimitry Andric case MachineOperand::MO_BlockAddress: 3620b57cec5SDimitry Andric return LowerSymbolOperand( 3630b57cec5SDimitry Andric MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress())); 3640b57cec5SDimitry Andric case MachineOperand::MO_RegisterMask: 3650b57cec5SDimitry Andric // Ignore call clobbers. 366*0fca6ea1SDimitry Andric return MCOperand(); 3670b57cec5SDimitry Andric } 3680b57cec5SDimitry Andric } 3690b57cec5SDimitry Andric 3708bcb0991SDimitry Andric // Replace TAILJMP opcodes with their equivalent opcodes that have encoding 3718bcb0991SDimitry Andric // information. 3728bcb0991SDimitry Andric static unsigned convertTailJumpOpcode(unsigned Opcode) { 3738bcb0991SDimitry Andric switch (Opcode) { 3748bcb0991SDimitry Andric case X86::TAILJMPr: 3758bcb0991SDimitry Andric Opcode = X86::JMP32r; 3768bcb0991SDimitry Andric break; 3778bcb0991SDimitry Andric case X86::TAILJMPm: 3788bcb0991SDimitry Andric Opcode = X86::JMP32m; 3798bcb0991SDimitry Andric break; 3808bcb0991SDimitry Andric case X86::TAILJMPr64: 3818bcb0991SDimitry Andric Opcode = X86::JMP64r; 3828bcb0991SDimitry Andric break; 3838bcb0991SDimitry Andric case X86::TAILJMPm64: 3848bcb0991SDimitry Andric Opcode = X86::JMP64m; 3858bcb0991SDimitry Andric break; 3868bcb0991SDimitry Andric case X86::TAILJMPr64_REX: 3878bcb0991SDimitry Andric Opcode = X86::JMP64r_REX; 3888bcb0991SDimitry Andric break; 3898bcb0991SDimitry Andric case X86::TAILJMPm64_REX: 3908bcb0991SDimitry Andric Opcode = X86::JMP64m_REX; 3918bcb0991SDimitry Andric break; 3928bcb0991SDimitry Andric case X86::TAILJMPd: 3938bcb0991SDimitry Andric case X86::TAILJMPd64: 3948bcb0991SDimitry Andric Opcode = X86::JMP_1; 3958bcb0991SDimitry Andric break; 3968bcb0991SDimitry Andric case X86::TAILJMPd_CC: 3978bcb0991SDimitry Andric case X86::TAILJMPd64_CC: 3988bcb0991SDimitry Andric Opcode = X86::JCC_1; 3998bcb0991SDimitry Andric break; 4008bcb0991SDimitry Andric } 4018bcb0991SDimitry Andric 4028bcb0991SDimitry Andric return Opcode; 4038bcb0991SDimitry Andric } 4048bcb0991SDimitry Andric 4050b57cec5SDimitry Andric void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const { 4060b57cec5SDimitry Andric OutMI.setOpcode(MI->getOpcode()); 4070b57cec5SDimitry Andric 4080b57cec5SDimitry Andric for (const MachineOperand &MO : MI->operands()) 409*0fca6ea1SDimitry Andric if (auto Op = LowerMachineOperand(MI, MO); Op.isValid()) 410*0fca6ea1SDimitry Andric OutMI.addOperand(Op); 4110b57cec5SDimitry Andric 41206c3fb27SDimitry Andric bool In64BitMode = AsmPrinter.getSubtarget().is64Bit(); 41306c3fb27SDimitry Andric if (X86::optimizeInstFromVEX3ToVEX2(OutMI, MI->getDesc()) || 41406c3fb27SDimitry Andric X86::optimizeShiftRotateWithImmediateOne(OutMI) || 41506c3fb27SDimitry Andric X86::optimizeVPCMPWithImmediateOneOrSix(OutMI) || 41606c3fb27SDimitry Andric X86::optimizeMOVSX(OutMI) || X86::optimizeINCDEC(OutMI, In64BitMode) || 41706c3fb27SDimitry Andric X86::optimizeMOV(OutMI, In64BitMode) || 41806c3fb27SDimitry Andric X86::optimizeToFixedRegisterOrShortImmediateForm(OutMI)) 41906c3fb27SDimitry Andric return; 42006c3fb27SDimitry Andric 4210b57cec5SDimitry Andric // Handle a few special cases to eliminate operand modifiers. 4220b57cec5SDimitry Andric switch (OutMI.getOpcode()) { 4230b57cec5SDimitry Andric case X86::LEA64_32r: 4240b57cec5SDimitry Andric case X86::LEA64r: 4250b57cec5SDimitry Andric case X86::LEA16r: 4260b57cec5SDimitry Andric case X86::LEA32r: 4270b57cec5SDimitry Andric // LEA should have a segment register, but it must be empty. 4280b57cec5SDimitry Andric assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands && 4290b57cec5SDimitry Andric "Unexpected # of LEA operands"); 4300b57cec5SDimitry Andric assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 && 4310b57cec5SDimitry Andric "LEA has segment specified!"); 4320b57cec5SDimitry Andric break; 4335ffd83dbSDimitry Andric case X86::MULX32Hrr: 4345ffd83dbSDimitry Andric case X86::MULX32Hrm: 4355ffd83dbSDimitry Andric case X86::MULX64Hrr: 4365ffd83dbSDimitry Andric case X86::MULX64Hrm: { 4375ffd83dbSDimitry Andric // Turn into regular MULX by duplicating the destination. 4385ffd83dbSDimitry Andric unsigned NewOpc; 4395ffd83dbSDimitry Andric switch (OutMI.getOpcode()) { 4405ffd83dbSDimitry Andric default: llvm_unreachable("Invalid opcode"); 4415ffd83dbSDimitry Andric case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break; 4425ffd83dbSDimitry Andric case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break; 4435ffd83dbSDimitry Andric case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break; 4445ffd83dbSDimitry Andric case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break; 4455ffd83dbSDimitry Andric } 4465ffd83dbSDimitry Andric OutMI.setOpcode(NewOpc); 4475ffd83dbSDimitry Andric // Duplicate the destination. 4485ffd83dbSDimitry Andric unsigned DestReg = OutMI.getOperand(0).getReg(); 4495ffd83dbSDimitry Andric OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg)); 4505ffd83dbSDimitry Andric break; 4515ffd83dbSDimitry Andric } 4528bcb0991SDimitry Andric // CALL64r, CALL64pcrel32 - These instructions used to have 4538bcb0991SDimitry Andric // register inputs modeled as normal uses instead of implicit uses. As such, 4548bcb0991SDimitry Andric // they we used to truncate off all but the first operand (the callee). This 4558bcb0991SDimitry Andric // issue seems to have been fixed at some point. This assert verifies that. 4568bcb0991SDimitry Andric case X86::CALL64r: 4578bcb0991SDimitry Andric case X86::CALL64pcrel32: 4588bcb0991SDimitry Andric assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!"); 4598bcb0991SDimitry Andric break; 4600b57cec5SDimitry Andric case X86::EH_RETURN: 4610b57cec5SDimitry Andric case X86::EH_RETURN64: { 4620b57cec5SDimitry Andric OutMI = MCInst(); 4630b57cec5SDimitry Andric OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); 4640b57cec5SDimitry Andric break; 4650b57cec5SDimitry Andric } 4660b57cec5SDimitry Andric case X86::CLEANUPRET: { 4670b57cec5SDimitry Andric // Replace CLEANUPRET with the appropriate RET. 4680b57cec5SDimitry Andric OutMI = MCInst(); 4690b57cec5SDimitry Andric OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget())); 4700b57cec5SDimitry Andric break; 4710b57cec5SDimitry Andric } 4720b57cec5SDimitry Andric case X86::CATCHRET: { 4730b57cec5SDimitry Andric // Replace CATCHRET with the appropriate RET. 4740b57cec5SDimitry Andric const X86Subtarget &Subtarget = AsmPrinter.getSubtarget(); 47506c3fb27SDimitry Andric unsigned ReturnReg = In64BitMode ? X86::RAX : X86::EAX; 4760b57cec5SDimitry Andric OutMI = MCInst(); 4770b57cec5SDimitry Andric OutMI.setOpcode(getRetOpcode(Subtarget)); 4780b57cec5SDimitry Andric OutMI.addOperand(MCOperand::createReg(ReturnReg)); 4790b57cec5SDimitry Andric break; 4800b57cec5SDimitry Andric } 4810b57cec5SDimitry Andric // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump 4820b57cec5SDimitry Andric // instruction. 4830b57cec5SDimitry Andric case X86::TAILJMPr: 4848bcb0991SDimitry Andric case X86::TAILJMPr64: 4858bcb0991SDimitry Andric case X86::TAILJMPr64_REX: 4860b57cec5SDimitry Andric case X86::TAILJMPd: 4870b57cec5SDimitry Andric case X86::TAILJMPd64: 4888bcb0991SDimitry Andric assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!"); 4898bcb0991SDimitry Andric OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode())); 4900b57cec5SDimitry Andric break; 4910b57cec5SDimitry Andric case X86::TAILJMPd_CC: 4928bcb0991SDimitry Andric case X86::TAILJMPd64_CC: 4938bcb0991SDimitry Andric assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!"); 4948bcb0991SDimitry Andric OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode())); 4950b57cec5SDimitry Andric break; 4968bcb0991SDimitry Andric case X86::TAILJMPm: 4978bcb0991SDimitry Andric case X86::TAILJMPm64: 4988bcb0991SDimitry Andric case X86::TAILJMPm64_REX: 4998bcb0991SDimitry Andric assert(OutMI.getNumOperands() == X86::AddrNumOperands && 5008bcb0991SDimitry Andric "Unexpected number of operands!"); 5018bcb0991SDimitry Andric OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode())); 5028bcb0991SDimitry Andric break; 50381ad6265SDimitry Andric case X86::MASKMOVDQU: 50481ad6265SDimitry Andric case X86::VMASKMOVDQU: 50506c3fb27SDimitry Andric if (In64BitMode) 50681ad6265SDimitry Andric OutMI.setFlags(X86::IP_HAS_AD_SIZE); 50781ad6265SDimitry Andric break; 50806c3fb27SDimitry Andric case X86::BSF16rm: 50906c3fb27SDimitry Andric case X86::BSF16rr: 51006c3fb27SDimitry Andric case X86::BSF32rm: 51106c3fb27SDimitry Andric case X86::BSF32rr: 51206c3fb27SDimitry Andric case X86::BSF64rm: 51306c3fb27SDimitry Andric case X86::BSF64rr: { 514bdd1243dSDimitry Andric // Add an REP prefix to BSF instructions so that new processors can 515bdd1243dSDimitry Andric // recognize as TZCNT, which has better performance than BSF. 516bdd1243dSDimitry Andric // BSF and TZCNT have different interpretations on ZF bit. So make sure 517bdd1243dSDimitry Andric // it won't be used later. 518*0fca6ea1SDimitry Andric const MachineOperand *FlagDef = 519*0fca6ea1SDimitry Andric MI->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr); 52006c3fb27SDimitry Andric if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead()) 521bdd1243dSDimitry Andric OutMI.setFlags(X86::IP_HAS_REPEAT); 522480093f4SDimitry Andric break; 523480093f4SDimitry Andric } 52406c3fb27SDimitry Andric default: 52506c3fb27SDimitry Andric break; 5260b57cec5SDimitry Andric } 5270b57cec5SDimitry Andric } 5280b57cec5SDimitry Andric 5290b57cec5SDimitry Andric void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering, 5300b57cec5SDimitry Andric const MachineInstr &MI) { 5315ffd83dbSDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 532*0fca6ea1SDimitry Andric bool Is64Bits = getSubtarget().is64Bit(); 533*0fca6ea1SDimitry Andric bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64(); 5340b57cec5SDimitry Andric MCContext &Ctx = OutStreamer->getContext(); 5350b57cec5SDimitry Andric 5360b57cec5SDimitry Andric MCSymbolRefExpr::VariantKind SRVK; 5370b57cec5SDimitry Andric switch (MI.getOpcode()) { 5380b57cec5SDimitry Andric case X86::TLS_addr32: 5390b57cec5SDimitry Andric case X86::TLS_addr64: 540e8d8bef9SDimitry Andric case X86::TLS_addrX32: 5410b57cec5SDimitry Andric SRVK = MCSymbolRefExpr::VK_TLSGD; 5420b57cec5SDimitry Andric break; 5430b57cec5SDimitry Andric case X86::TLS_base_addr32: 5440b57cec5SDimitry Andric SRVK = MCSymbolRefExpr::VK_TLSLDM; 5450b57cec5SDimitry Andric break; 5460b57cec5SDimitry Andric case X86::TLS_base_addr64: 547e8d8bef9SDimitry Andric case X86::TLS_base_addrX32: 5480b57cec5SDimitry Andric SRVK = MCSymbolRefExpr::VK_TLSLD; 5490b57cec5SDimitry Andric break; 550*0fca6ea1SDimitry Andric case X86::TLS_desc32: 551*0fca6ea1SDimitry Andric case X86::TLS_desc64: 552*0fca6ea1SDimitry Andric SRVK = MCSymbolRefExpr::VK_TLSDESC; 553*0fca6ea1SDimitry Andric break; 5540b57cec5SDimitry Andric default: 5550b57cec5SDimitry Andric llvm_unreachable("unexpected opcode"); 5560b57cec5SDimitry Andric } 5570b57cec5SDimitry Andric 5580b57cec5SDimitry Andric const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create( 5590b57cec5SDimitry Andric MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx); 5600b57cec5SDimitry Andric 561*0fca6ea1SDimitry Andric // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD 5620b57cec5SDimitry Andric // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is 5630b57cec5SDimitry Andric // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by 5640b57cec5SDimitry Andric // only using GOT when GOTPCRELX is enabled. 565*0fca6ea1SDimitry Andric // TODO Delete the workaround when rustc no longer relies on the hack 5660b57cec5SDimitry Andric bool UseGot = MMI->getModule()->getRtLibUseGOT() && 567*0fca6ea1SDimitry Andric Ctx.getTargetOptions()->X86RelaxRelocations; 5680b57cec5SDimitry Andric 569*0fca6ea1SDimitry Andric if (SRVK == MCSymbolRefExpr::VK_TLSDESC) { 570*0fca6ea1SDimitry Andric const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create( 571*0fca6ea1SDimitry Andric MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), 572*0fca6ea1SDimitry Andric MCSymbolRefExpr::VK_TLSCALL, Ctx); 573*0fca6ea1SDimitry Andric EmitAndCountInstruction( 574*0fca6ea1SDimitry Andric MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r) 575*0fca6ea1SDimitry Andric .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX) 576*0fca6ea1SDimitry Andric .addReg(Is64Bits ? X86::RIP : X86::EBX) 577*0fca6ea1SDimitry Andric .addImm(1) 578*0fca6ea1SDimitry Andric .addReg(0) 579*0fca6ea1SDimitry Andric .addExpr(Sym) 580*0fca6ea1SDimitry Andric .addReg(0)); 581*0fca6ea1SDimitry Andric EmitAndCountInstruction( 582*0fca6ea1SDimitry Andric MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m) 583*0fca6ea1SDimitry Andric .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX) 584*0fca6ea1SDimitry Andric .addImm(1) 585*0fca6ea1SDimitry Andric .addReg(0) 586*0fca6ea1SDimitry Andric .addExpr(Expr) 587*0fca6ea1SDimitry Andric .addReg(0)); 588*0fca6ea1SDimitry Andric } else if (Is64Bits) { 5890b57cec5SDimitry Andric bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD; 590e8d8bef9SDimitry Andric if (NeedsPadding && Is64BitsLP64) 5910b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); 5920b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::LEA64r) 5930b57cec5SDimitry Andric .addReg(X86::RDI) 5940b57cec5SDimitry Andric .addReg(X86::RIP) 5950b57cec5SDimitry Andric .addImm(1) 5960b57cec5SDimitry Andric .addReg(0) 5970b57cec5SDimitry Andric .addExpr(Sym) 5980b57cec5SDimitry Andric .addReg(0)); 5990b57cec5SDimitry Andric const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr"); 6000b57cec5SDimitry Andric if (NeedsPadding) { 6010b57cec5SDimitry Andric if (!UseGot) 6020b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); 6030b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX)); 6040b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX)); 6050b57cec5SDimitry Andric } 6060b57cec5SDimitry Andric if (UseGot) { 6070b57cec5SDimitry Andric const MCExpr *Expr = MCSymbolRefExpr::create( 6080b57cec5SDimitry Andric TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx); 6090b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CALL64m) 6100b57cec5SDimitry Andric .addReg(X86::RIP) 6110b57cec5SDimitry Andric .addImm(1) 6120b57cec5SDimitry Andric .addReg(0) 6130b57cec5SDimitry Andric .addExpr(Expr) 6140b57cec5SDimitry Andric .addReg(0)); 6150b57cec5SDimitry Andric } else { 6160b57cec5SDimitry Andric EmitAndCountInstruction( 6170b57cec5SDimitry Andric MCInstBuilder(X86::CALL64pcrel32) 6180b57cec5SDimitry Andric .addExpr(MCSymbolRefExpr::create(TlsGetAddr, 6190b57cec5SDimitry Andric MCSymbolRefExpr::VK_PLT, Ctx))); 6200b57cec5SDimitry Andric } 6210b57cec5SDimitry Andric } else { 6220b57cec5SDimitry Andric if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) { 6230b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::LEA32r) 6240b57cec5SDimitry Andric .addReg(X86::EAX) 6250b57cec5SDimitry Andric .addReg(0) 6260b57cec5SDimitry Andric .addImm(1) 6270b57cec5SDimitry Andric .addReg(X86::EBX) 6280b57cec5SDimitry Andric .addExpr(Sym) 6290b57cec5SDimitry Andric .addReg(0)); 6300b57cec5SDimitry Andric } else { 6310b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::LEA32r) 6320b57cec5SDimitry Andric .addReg(X86::EAX) 6330b57cec5SDimitry Andric .addReg(X86::EBX) 6340b57cec5SDimitry Andric .addImm(1) 6350b57cec5SDimitry Andric .addReg(0) 6360b57cec5SDimitry Andric .addExpr(Sym) 6370b57cec5SDimitry Andric .addReg(0)); 6380b57cec5SDimitry Andric } 6390b57cec5SDimitry Andric 6400b57cec5SDimitry Andric const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr"); 6410b57cec5SDimitry Andric if (UseGot) { 6420b57cec5SDimitry Andric const MCExpr *Expr = 6430b57cec5SDimitry Andric MCSymbolRefExpr::create(TlsGetAddr, MCSymbolRefExpr::VK_GOT, Ctx); 6440b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CALL32m) 6450b57cec5SDimitry Andric .addReg(X86::EBX) 6460b57cec5SDimitry Andric .addImm(1) 6470b57cec5SDimitry Andric .addReg(0) 6480b57cec5SDimitry Andric .addExpr(Expr) 6490b57cec5SDimitry Andric .addReg(0)); 6500b57cec5SDimitry Andric } else { 6510b57cec5SDimitry Andric EmitAndCountInstruction( 6520b57cec5SDimitry Andric MCInstBuilder(X86::CALLpcrel32) 6530b57cec5SDimitry Andric .addExpr(MCSymbolRefExpr::create(TlsGetAddr, 6540b57cec5SDimitry Andric MCSymbolRefExpr::VK_PLT, Ctx))); 6550b57cec5SDimitry Andric } 6560b57cec5SDimitry Andric } 6570b57cec5SDimitry Andric } 6580b57cec5SDimitry Andric 6590b57cec5SDimitry Andric /// Emit the largest nop instruction smaller than or equal to \p NumBytes 6600b57cec5SDimitry Andric /// bytes. Return the size of nop emitted. 6615ffd83dbSDimitry Andric static unsigned emitNop(MCStreamer &OS, unsigned NumBytes, 6625ffd83dbSDimitry Andric const X86Subtarget *Subtarget) { 663e8d8bef9SDimitry Andric // Determine the longest nop which can be efficiently decoded for the given 664e8d8bef9SDimitry Andric // target cpu. 15-bytes is the longest single NOP instruction, but some 665e8d8bef9SDimitry Andric // platforms can't decode the longest forms efficiently. 666e8d8bef9SDimitry Andric unsigned MaxNopLength = 1; 667e8d8bef9SDimitry Andric if (Subtarget->is64Bit()) { 668e8d8bef9SDimitry Andric // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the 669e8d8bef9SDimitry Andric // IndexReg/BaseReg below need to be updated. 670349cc55cSDimitry Andric if (Subtarget->hasFeature(X86::TuningFast7ByteNOP)) 671e8d8bef9SDimitry Andric MaxNopLength = 7; 672349cc55cSDimitry Andric else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP)) 673e8d8bef9SDimitry Andric MaxNopLength = 15; 674349cc55cSDimitry Andric else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP)) 675e8d8bef9SDimitry Andric MaxNopLength = 11; 676e8d8bef9SDimitry Andric else 677e8d8bef9SDimitry Andric MaxNopLength = 10; 678e8d8bef9SDimitry Andric } if (Subtarget->is32Bit()) 679e8d8bef9SDimitry Andric MaxNopLength = 2; 680e8d8bef9SDimitry Andric 681480093f4SDimitry Andric // Cap a single nop emission at the profitable value for the target 682e8d8bef9SDimitry Andric NumBytes = std::min(NumBytes, MaxNopLength); 6830b57cec5SDimitry Andric 6840b57cec5SDimitry Andric unsigned NopSize; 6850b57cec5SDimitry Andric unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg; 6860b57cec5SDimitry Andric IndexReg = Displacement = SegmentReg = 0; 6870b57cec5SDimitry Andric BaseReg = X86::RAX; 6880b57cec5SDimitry Andric ScaleVal = 1; 6890b57cec5SDimitry Andric switch (NumBytes) { 6900b57cec5SDimitry Andric case 0: 6910b57cec5SDimitry Andric llvm_unreachable("Zero nops?"); 6920b57cec5SDimitry Andric break; 6930b57cec5SDimitry Andric case 1: 6940b57cec5SDimitry Andric NopSize = 1; 6950b57cec5SDimitry Andric Opc = X86::NOOP; 6960b57cec5SDimitry Andric break; 6970b57cec5SDimitry Andric case 2: 6980b57cec5SDimitry Andric NopSize = 2; 6990b57cec5SDimitry Andric Opc = X86::XCHG16ar; 7000b57cec5SDimitry Andric break; 7010b57cec5SDimitry Andric case 3: 7020b57cec5SDimitry Andric NopSize = 3; 7030b57cec5SDimitry Andric Opc = X86::NOOPL; 7040b57cec5SDimitry Andric break; 7050b57cec5SDimitry Andric case 4: 7060b57cec5SDimitry Andric NopSize = 4; 7070b57cec5SDimitry Andric Opc = X86::NOOPL; 7080b57cec5SDimitry Andric Displacement = 8; 7090b57cec5SDimitry Andric break; 7100b57cec5SDimitry Andric case 5: 7110b57cec5SDimitry Andric NopSize = 5; 7120b57cec5SDimitry Andric Opc = X86::NOOPL; 7130b57cec5SDimitry Andric Displacement = 8; 7140b57cec5SDimitry Andric IndexReg = X86::RAX; 7150b57cec5SDimitry Andric break; 7160b57cec5SDimitry Andric case 6: 7170b57cec5SDimitry Andric NopSize = 6; 7180b57cec5SDimitry Andric Opc = X86::NOOPW; 7190b57cec5SDimitry Andric Displacement = 8; 7200b57cec5SDimitry Andric IndexReg = X86::RAX; 7210b57cec5SDimitry Andric break; 7220b57cec5SDimitry Andric case 7: 7230b57cec5SDimitry Andric NopSize = 7; 7240b57cec5SDimitry Andric Opc = X86::NOOPL; 7250b57cec5SDimitry Andric Displacement = 512; 7260b57cec5SDimitry Andric break; 7270b57cec5SDimitry Andric case 8: 7280b57cec5SDimitry Andric NopSize = 8; 7290b57cec5SDimitry Andric Opc = X86::NOOPL; 7300b57cec5SDimitry Andric Displacement = 512; 7310b57cec5SDimitry Andric IndexReg = X86::RAX; 7320b57cec5SDimitry Andric break; 7330b57cec5SDimitry Andric case 9: 7340b57cec5SDimitry Andric NopSize = 9; 7350b57cec5SDimitry Andric Opc = X86::NOOPW; 7360b57cec5SDimitry Andric Displacement = 512; 7370b57cec5SDimitry Andric IndexReg = X86::RAX; 7380b57cec5SDimitry Andric break; 7390b57cec5SDimitry Andric default: 7400b57cec5SDimitry Andric NopSize = 10; 7410b57cec5SDimitry Andric Opc = X86::NOOPW; 7420b57cec5SDimitry Andric Displacement = 512; 7430b57cec5SDimitry Andric IndexReg = X86::RAX; 7440b57cec5SDimitry Andric SegmentReg = X86::CS; 7450b57cec5SDimitry Andric break; 7460b57cec5SDimitry Andric } 7470b57cec5SDimitry Andric 7480b57cec5SDimitry Andric unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U); 7490b57cec5SDimitry Andric NopSize += NumPrefixes; 7500b57cec5SDimitry Andric for (unsigned i = 0; i != NumPrefixes; ++i) 7515ffd83dbSDimitry Andric OS.emitBytes("\x66"); 7520b57cec5SDimitry Andric 7530b57cec5SDimitry Andric switch (Opc) { 7540b57cec5SDimitry Andric default: llvm_unreachable("Unexpected opcode"); 7550b57cec5SDimitry Andric case X86::NOOP: 7565ffd83dbSDimitry Andric OS.emitInstruction(MCInstBuilder(Opc), *Subtarget); 7570b57cec5SDimitry Andric break; 7580b57cec5SDimitry Andric case X86::XCHG16ar: 7595ffd83dbSDimitry Andric OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX), 7605ffd83dbSDimitry Andric *Subtarget); 7610b57cec5SDimitry Andric break; 7620b57cec5SDimitry Andric case X86::NOOPL: 7630b57cec5SDimitry Andric case X86::NOOPW: 7645ffd83dbSDimitry Andric OS.emitInstruction(MCInstBuilder(Opc) 7650b57cec5SDimitry Andric .addReg(BaseReg) 7660b57cec5SDimitry Andric .addImm(ScaleVal) 7670b57cec5SDimitry Andric .addReg(IndexReg) 7680b57cec5SDimitry Andric .addImm(Displacement) 7690b57cec5SDimitry Andric .addReg(SegmentReg), 7705ffd83dbSDimitry Andric *Subtarget); 7710b57cec5SDimitry Andric break; 7720b57cec5SDimitry Andric } 7730b57cec5SDimitry Andric assert(NopSize <= NumBytes && "We overemitted?"); 7740b57cec5SDimitry Andric return NopSize; 7750b57cec5SDimitry Andric } 7760b57cec5SDimitry Andric 7770b57cec5SDimitry Andric /// Emit the optimal amount of multi-byte nops on X86. 7785ffd83dbSDimitry Andric static void emitX86Nops(MCStreamer &OS, unsigned NumBytes, 7795ffd83dbSDimitry Andric const X86Subtarget *Subtarget) { 7800b57cec5SDimitry Andric unsigned NopsToEmit = NumBytes; 7810b57cec5SDimitry Andric (void)NopsToEmit; 7820b57cec5SDimitry Andric while (NumBytes) { 7835ffd83dbSDimitry Andric NumBytes -= emitNop(OS, NumBytes, Subtarget); 7840b57cec5SDimitry Andric assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!"); 7850b57cec5SDimitry Andric } 7860b57cec5SDimitry Andric } 7870b57cec5SDimitry Andric 7880b57cec5SDimitry Andric void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI, 7890b57cec5SDimitry Andric X86MCInstLower &MCIL) { 7900b57cec5SDimitry Andric assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64"); 7910b57cec5SDimitry Andric 792480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 793480093f4SDimitry Andric 7940b57cec5SDimitry Andric StatepointOpers SOpers(&MI); 7950b57cec5SDimitry Andric if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { 7965ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, PatchBytes, Subtarget); 7970b57cec5SDimitry Andric } else { 7980b57cec5SDimitry Andric // Lower call target and choose correct opcode 7990b57cec5SDimitry Andric const MachineOperand &CallTarget = SOpers.getCallTarget(); 8000b57cec5SDimitry Andric MCOperand CallTargetMCOp; 8010b57cec5SDimitry Andric unsigned CallOpcode; 8020b57cec5SDimitry Andric switch (CallTarget.getType()) { 8030b57cec5SDimitry Andric case MachineOperand::MO_GlobalAddress: 8040b57cec5SDimitry Andric case MachineOperand::MO_ExternalSymbol: 8050b57cec5SDimitry Andric CallTargetMCOp = MCIL.LowerSymbolOperand( 8060b57cec5SDimitry Andric CallTarget, MCIL.GetSymbolFromOperand(CallTarget)); 8070b57cec5SDimitry Andric CallOpcode = X86::CALL64pcrel32; 8080b57cec5SDimitry Andric // Currently, we only support relative addressing with statepoints. 8090b57cec5SDimitry Andric // Otherwise, we'll need a scratch register to hold the target 8100b57cec5SDimitry Andric // address. You'll fail asserts during load & relocation if this 8110b57cec5SDimitry Andric // symbol is to far away. (TODO: support non-relative addressing) 8120b57cec5SDimitry Andric break; 8130b57cec5SDimitry Andric case MachineOperand::MO_Immediate: 8140b57cec5SDimitry Andric CallTargetMCOp = MCOperand::createImm(CallTarget.getImm()); 8150b57cec5SDimitry Andric CallOpcode = X86::CALL64pcrel32; 8160b57cec5SDimitry Andric // Currently, we only support relative addressing with statepoints. 8170b57cec5SDimitry Andric // Otherwise, we'll need a scratch register to hold the target 8180b57cec5SDimitry Andric // immediate. You'll fail asserts during load & relocation if this 8190b57cec5SDimitry Andric // address is to far away. (TODO: support non-relative addressing) 8200b57cec5SDimitry Andric break; 8210b57cec5SDimitry Andric case MachineOperand::MO_Register: 8220b57cec5SDimitry Andric // FIXME: Add retpoline support and remove this. 8230946e70aSDimitry Andric if (Subtarget->useIndirectThunkCalls()) 8240946e70aSDimitry Andric report_fatal_error("Lowering register statepoints with thunks not " 8250b57cec5SDimitry Andric "yet implemented."); 8260b57cec5SDimitry Andric CallTargetMCOp = MCOperand::createReg(CallTarget.getReg()); 8270b57cec5SDimitry Andric CallOpcode = X86::CALL64r; 8280b57cec5SDimitry Andric break; 8290b57cec5SDimitry Andric default: 8300b57cec5SDimitry Andric llvm_unreachable("Unsupported operand type in statepoint call target"); 8310b57cec5SDimitry Andric break; 8320b57cec5SDimitry Andric } 8330b57cec5SDimitry Andric 8340b57cec5SDimitry Andric // Emit call 8350b57cec5SDimitry Andric MCInst CallInst; 8360b57cec5SDimitry Andric CallInst.setOpcode(CallOpcode); 8370b57cec5SDimitry Andric CallInst.addOperand(CallTargetMCOp); 8385ffd83dbSDimitry Andric OutStreamer->emitInstruction(CallInst, getSubtargetInfo()); 8390b57cec5SDimitry Andric } 8400b57cec5SDimitry Andric 8410b57cec5SDimitry Andric // Record our statepoint node in the same section used by STACKMAP 8420b57cec5SDimitry Andric // and PATCHPOINT 843480093f4SDimitry Andric auto &Ctx = OutStreamer->getContext(); 844480093f4SDimitry Andric MCSymbol *MILabel = Ctx.createTempSymbol(); 8455ffd83dbSDimitry Andric OutStreamer->emitLabel(MILabel); 846480093f4SDimitry Andric SM.recordStatepoint(*MILabel, MI); 8470b57cec5SDimitry Andric } 8480b57cec5SDimitry Andric 8490b57cec5SDimitry Andric void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI, 8500b57cec5SDimitry Andric X86MCInstLower &MCIL) { 8510b57cec5SDimitry Andric // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>, 8520b57cec5SDimitry Andric // <opcode>, <operands> 8530b57cec5SDimitry Andric 854480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 855480093f4SDimitry Andric 8568bcb0991SDimitry Andric Register DefRegister = FaultingMI.getOperand(0).getReg(); 8570b57cec5SDimitry Andric FaultMaps::FaultKind FK = 8580b57cec5SDimitry Andric static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm()); 8590b57cec5SDimitry Andric MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol(); 8600b57cec5SDimitry Andric unsigned Opcode = FaultingMI.getOperand(3).getImm(); 8610b57cec5SDimitry Andric unsigned OperandsBeginIdx = 4; 8620b57cec5SDimitry Andric 863480093f4SDimitry Andric auto &Ctx = OutStreamer->getContext(); 864480093f4SDimitry Andric MCSymbol *FaultingLabel = Ctx.createTempSymbol(); 8655ffd83dbSDimitry Andric OutStreamer->emitLabel(FaultingLabel); 866480093f4SDimitry Andric 8670b57cec5SDimitry Andric assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!"); 868480093f4SDimitry Andric FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel); 8690b57cec5SDimitry Andric 8700b57cec5SDimitry Andric MCInst MI; 8710b57cec5SDimitry Andric MI.setOpcode(Opcode); 8720b57cec5SDimitry Andric 8730b57cec5SDimitry Andric if (DefRegister != X86::NoRegister) 8740b57cec5SDimitry Andric MI.addOperand(MCOperand::createReg(DefRegister)); 8750b57cec5SDimitry Andric 876bdd1243dSDimitry Andric for (const MachineOperand &MO : 877bdd1243dSDimitry Andric llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) 878*0fca6ea1SDimitry Andric if (auto Op = MCIL.LowerMachineOperand(&FaultingMI, MO); Op.isValid()) 879*0fca6ea1SDimitry Andric MI.addOperand(Op); 8800b57cec5SDimitry Andric 8810b57cec5SDimitry Andric OutStreamer->AddComment("on-fault: " + HandlerLabel->getName()); 8825ffd83dbSDimitry Andric OutStreamer->emitInstruction(MI, getSubtargetInfo()); 8830b57cec5SDimitry Andric } 8840b57cec5SDimitry Andric 8850b57cec5SDimitry Andric void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI, 8860b57cec5SDimitry Andric X86MCInstLower &MCIL) { 8870b57cec5SDimitry Andric bool Is64Bits = Subtarget->is64Bit(); 8880b57cec5SDimitry Andric MCContext &Ctx = OutStreamer->getContext(); 8890b57cec5SDimitry Andric MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__"); 8900b57cec5SDimitry Andric const MCSymbolRefExpr *Op = 8910b57cec5SDimitry Andric MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_None, Ctx); 8920b57cec5SDimitry Andric 8930b57cec5SDimitry Andric EmitAndCountInstruction( 8940b57cec5SDimitry Andric MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32) 8950b57cec5SDimitry Andric .addExpr(Op)); 8960b57cec5SDimitry Andric } 8970b57cec5SDimitry Andric 898bdd1243dSDimitry Andric void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { 899bdd1243dSDimitry Andric assert(std::next(MI.getIterator())->isCall() && 900bdd1243dSDimitry Andric "KCFI_CHECK not followed by a call instruction"); 901bdd1243dSDimitry Andric 902bdd1243dSDimitry Andric // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop() 903bdd1243dSDimitry Andric // returns a 1-byte X86::NOOP, which means the offset is the same in 904bdd1243dSDimitry Andric // bytes. This assumes that patchable-function-prefix is the same for all 905bdd1243dSDimitry Andric // functions. 906bdd1243dSDimitry Andric const MachineFunction &MF = *MI.getMF(); 907bdd1243dSDimitry Andric int64_t PrefixNops = 0; 908bdd1243dSDimitry Andric (void)MF.getFunction() 909bdd1243dSDimitry Andric .getFnAttribute("patchable-function-prefix") 910bdd1243dSDimitry Andric .getValueAsString() 911bdd1243dSDimitry Andric .getAsInteger(10, PrefixNops); 912bdd1243dSDimitry Andric 913bdd1243dSDimitry Andric // KCFI allows indirect calls to any location that's preceded by a valid 914bdd1243dSDimitry Andric // type identifier. To avoid encoding the full constant into an instruction, 915bdd1243dSDimitry Andric // and thus emitting potential call target gadgets at each indirect call 916bdd1243dSDimitry Andric // site, load a negated constant to a register and compare that to the 917bdd1243dSDimitry Andric // expected value at the call target. 918bdd1243dSDimitry Andric const Register AddrReg = MI.getOperand(0).getReg(); 919bdd1243dSDimitry Andric const uint32_t Type = MI.getOperand(1).getImm(); 920bdd1243dSDimitry Andric // The check is immediately before the call. If the call target is in R10, 921bdd1243dSDimitry Andric // we can clobber R11 for the check instead. 922bdd1243dSDimitry Andric unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D; 923bdd1243dSDimitry Andric EmitAndCountInstruction( 924bdd1243dSDimitry Andric MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type))); 925bdd1243dSDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::ADD32rm) 926bdd1243dSDimitry Andric .addReg(X86::NoRegister) 927bdd1243dSDimitry Andric .addReg(TempReg) 928bdd1243dSDimitry Andric .addReg(AddrReg) 929bdd1243dSDimitry Andric .addImm(1) 930bdd1243dSDimitry Andric .addReg(X86::NoRegister) 931bdd1243dSDimitry Andric .addImm(-(PrefixNops + 4)) 932bdd1243dSDimitry Andric .addReg(X86::NoRegister)); 933bdd1243dSDimitry Andric 934bdd1243dSDimitry Andric MCSymbol *Pass = OutContext.createTempSymbol(); 935bdd1243dSDimitry Andric EmitAndCountInstruction( 936bdd1243dSDimitry Andric MCInstBuilder(X86::JCC_1) 937bdd1243dSDimitry Andric .addExpr(MCSymbolRefExpr::create(Pass, OutContext)) 938bdd1243dSDimitry Andric .addImm(X86::COND_E)); 939bdd1243dSDimitry Andric 940bdd1243dSDimitry Andric MCSymbol *Trap = OutContext.createTempSymbol(); 941bdd1243dSDimitry Andric OutStreamer->emitLabel(Trap); 942bdd1243dSDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::TRAP)); 943bdd1243dSDimitry Andric emitKCFITrapEntry(MF, Trap); 944bdd1243dSDimitry Andric OutStreamer->emitLabel(Pass); 945bdd1243dSDimitry Andric } 946bdd1243dSDimitry Andric 947349cc55cSDimitry Andric void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) { 948349cc55cSDimitry Andric // FIXME: Make this work on non-ELF. 949349cc55cSDimitry Andric if (!TM.getTargetTriple().isOSBinFormatELF()) { 950349cc55cSDimitry Andric report_fatal_error("llvm.asan.check.memaccess only supported on ELF"); 951349cc55cSDimitry Andric return; 952349cc55cSDimitry Andric } 953349cc55cSDimitry Andric 9540eae32dcSDimitry Andric const auto &Reg = MI.getOperand(0).getReg(); 955349cc55cSDimitry Andric ASanAccessInfo AccessInfo(MI.getOperand(1).getImm()); 956349cc55cSDimitry Andric 9570eae32dcSDimitry Andric uint64_t ShadowBase; 9580eae32dcSDimitry Andric int MappingScale; 9590eae32dcSDimitry Andric bool OrShadowOffset; 9600eae32dcSDimitry Andric getAddressSanitizerParams(Triple(TM.getTargetTriple()), 64, 9610eae32dcSDimitry Andric AccessInfo.CompileKernel, &ShadowBase, 9620eae32dcSDimitry Andric &MappingScale, &OrShadowOffset); 9630eae32dcSDimitry Andric 96481ad6265SDimitry Andric StringRef Name = AccessInfo.IsWrite ? "store" : "load"; 96581ad6265SDimitry Andric StringRef Op = OrShadowOffset ? "or" : "add"; 96681ad6265SDimitry Andric std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" + 96781ad6265SDimitry Andric Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" + 96881ad6265SDimitry Andric TM.getMCRegisterInfo()->getName(Reg.asMCReg())) 96981ad6265SDimitry Andric .str(); 9700eae32dcSDimitry Andric if (OrShadowOffset) 9710eae32dcSDimitry Andric report_fatal_error( 9720eae32dcSDimitry Andric "OrShadowOffset is not supported with optimized callbacks"); 973349cc55cSDimitry Andric 974349cc55cSDimitry Andric EmitAndCountInstruction( 975349cc55cSDimitry Andric MCInstBuilder(X86::CALL64pcrel32) 9760eae32dcSDimitry Andric .addExpr(MCSymbolRefExpr::create( 9770eae32dcSDimitry Andric OutContext.getOrCreateSymbol(SymName), OutContext))); 978349cc55cSDimitry Andric } 979349cc55cSDimitry Andric 9800b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI, 9810b57cec5SDimitry Andric X86MCInstLower &MCIL) { 9827a6dacacSDimitry Andric // PATCHABLE_OP minsize 9830b57cec5SDimitry Andric 984480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 985480093f4SDimitry Andric 9867a6dacacSDimitry Andric auto NextMI = std::find_if(std::next(MI.getIterator()), 9877a6dacacSDimitry Andric MI.getParent()->end().getInstrIterator(), 9887a6dacacSDimitry Andric [](auto &II) { return !II.isMetaInstruction(); }); 9890b57cec5SDimitry Andric 9900b57cec5SDimitry Andric SmallString<256> Code; 9917a6dacacSDimitry Andric unsigned MinSize = MI.getOperand(0).getImm(); 9927a6dacacSDimitry Andric 993dfa39133SDimitry Andric if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) { 9947a6dacacSDimitry Andric // Lower the next MachineInstr to find its byte size. 995dfa39133SDimitry Andric // If the next instruction is inline assembly, we skip lowering it for now, 996dfa39133SDimitry Andric // and assume we should always generate NOPs. 9977a6dacacSDimitry Andric MCInst MCI; 9987a6dacacSDimitry Andric MCIL.Lower(&*NextMI, MCI); 9997a6dacacSDimitry Andric 10000b57cec5SDimitry Andric SmallVector<MCFixup, 4> Fixups; 100106c3fb27SDimitry Andric CodeEmitter->encodeInstruction(MCI, Code, Fixups, getSubtargetInfo()); 1002bdd1243dSDimitry Andric } 10030b57cec5SDimitry Andric 10040b57cec5SDimitry Andric if (Code.size() < MinSize) { 10055ffd83dbSDimitry Andric if (MinSize == 2 && Subtarget->is32Bit() && 10065ffd83dbSDimitry Andric Subtarget->isTargetWindowsMSVC() && 10075ffd83dbSDimitry Andric (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) { 10085f757f3fSDimitry Andric // For compatibility reasons, when targetting MSVC, it is important to 10095ffd83dbSDimitry Andric // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools 10105ffd83dbSDimitry Andric // rely specifically on this pattern to be able to patch a function. 10115ffd83dbSDimitry Andric // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE. 10125ffd83dbSDimitry Andric OutStreamer->emitInstruction( 10135ffd83dbSDimitry Andric MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI), 10145ffd83dbSDimitry Andric *Subtarget); 10150b57cec5SDimitry Andric } else { 10165ffd83dbSDimitry Andric unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget); 10170b57cec5SDimitry Andric assert(NopSize == MinSize && "Could not implement MinSize!"); 10180b57cec5SDimitry Andric (void)NopSize; 10190b57cec5SDimitry Andric } 10200b57cec5SDimitry Andric } 10210b57cec5SDimitry Andric } 10220b57cec5SDimitry Andric 10230b57cec5SDimitry Andric // Lower a stackmap of the form: 10240b57cec5SDimitry Andric // <id>, <shadowBytes>, ... 10250b57cec5SDimitry Andric void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) { 10260b57cec5SDimitry Andric SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); 1027480093f4SDimitry Andric 1028480093f4SDimitry Andric auto &Ctx = OutStreamer->getContext(); 1029480093f4SDimitry Andric MCSymbol *MILabel = Ctx.createTempSymbol(); 10305ffd83dbSDimitry Andric OutStreamer->emitLabel(MILabel); 1031480093f4SDimitry Andric 1032480093f4SDimitry Andric SM.recordStackMap(*MILabel, MI); 10330b57cec5SDimitry Andric unsigned NumShadowBytes = MI.getOperand(1).getImm(); 10340b57cec5SDimitry Andric SMShadowTracker.reset(NumShadowBytes); 10350b57cec5SDimitry Andric } 10360b57cec5SDimitry Andric 10370b57cec5SDimitry Andric // Lower a patchpoint of the form: 10380b57cec5SDimitry Andric // [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ... 10390b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI, 10400b57cec5SDimitry Andric X86MCInstLower &MCIL) { 10410b57cec5SDimitry Andric assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64"); 10420b57cec5SDimitry Andric 10430b57cec5SDimitry Andric SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); 10440b57cec5SDimitry Andric 1045480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 1046480093f4SDimitry Andric 1047480093f4SDimitry Andric auto &Ctx = OutStreamer->getContext(); 1048480093f4SDimitry Andric MCSymbol *MILabel = Ctx.createTempSymbol(); 10495ffd83dbSDimitry Andric OutStreamer->emitLabel(MILabel); 1050480093f4SDimitry Andric SM.recordPatchPoint(*MILabel, MI); 10510b57cec5SDimitry Andric 10520b57cec5SDimitry Andric PatchPointOpers opers(&MI); 10530b57cec5SDimitry Andric unsigned ScratchIdx = opers.getNextScratchIdx(); 10540b57cec5SDimitry Andric unsigned EncodedBytes = 0; 10550b57cec5SDimitry Andric const MachineOperand &CalleeMO = opers.getCallTarget(); 10560b57cec5SDimitry Andric 10570b57cec5SDimitry Andric // Check for null target. If target is non-null (i.e. is non-zero or is 10580b57cec5SDimitry Andric // symbolic) then emit a call. 10590b57cec5SDimitry Andric if (!(CalleeMO.isImm() && !CalleeMO.getImm())) { 10600b57cec5SDimitry Andric MCOperand CalleeMCOp; 10610b57cec5SDimitry Andric switch (CalleeMO.getType()) { 10620b57cec5SDimitry Andric default: 10630b57cec5SDimitry Andric /// FIXME: Add a verifier check for bad callee types. 10640b57cec5SDimitry Andric llvm_unreachable("Unrecognized callee operand type."); 10650b57cec5SDimitry Andric case MachineOperand::MO_Immediate: 10660b57cec5SDimitry Andric if (CalleeMO.getImm()) 10670b57cec5SDimitry Andric CalleeMCOp = MCOperand::createImm(CalleeMO.getImm()); 10680b57cec5SDimitry Andric break; 10690b57cec5SDimitry Andric case MachineOperand::MO_ExternalSymbol: 10700b57cec5SDimitry Andric case MachineOperand::MO_GlobalAddress: 10710b57cec5SDimitry Andric CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO, 10720b57cec5SDimitry Andric MCIL.GetSymbolFromOperand(CalleeMO)); 10730b57cec5SDimitry Andric break; 10740b57cec5SDimitry Andric } 10750b57cec5SDimitry Andric 10760b57cec5SDimitry Andric // Emit MOV to materialize the target address and the CALL to target. 10770b57cec5SDimitry Andric // This is encoded with 12-13 bytes, depending on which register is used. 10788bcb0991SDimitry Andric Register ScratchReg = MI.getOperand(ScratchIdx).getReg(); 10790b57cec5SDimitry Andric if (X86II::isX86_64ExtendedReg(ScratchReg)) 10800b57cec5SDimitry Andric EncodedBytes = 13; 10810b57cec5SDimitry Andric else 10820b57cec5SDimitry Andric EncodedBytes = 12; 10830b57cec5SDimitry Andric 10840b57cec5SDimitry Andric EmitAndCountInstruction( 10850b57cec5SDimitry Andric MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp)); 10860b57cec5SDimitry Andric // FIXME: Add retpoline support and remove this. 10870946e70aSDimitry Andric if (Subtarget->useIndirectThunkCalls()) 10880b57cec5SDimitry Andric report_fatal_error( 10890946e70aSDimitry Andric "Lowering patchpoint with thunks not yet implemented."); 10900b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg)); 10910b57cec5SDimitry Andric } 10920b57cec5SDimitry Andric 10930b57cec5SDimitry Andric // Emit padding. 10940b57cec5SDimitry Andric unsigned NumBytes = opers.getNumPatchBytes(); 10950b57cec5SDimitry Andric assert(NumBytes >= EncodedBytes && 10960b57cec5SDimitry Andric "Patchpoint can't request size less than the length of a call."); 10970b57cec5SDimitry Andric 10985ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget); 10990b57cec5SDimitry Andric } 11000b57cec5SDimitry Andric 11010b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, 11020b57cec5SDimitry Andric X86MCInstLower &MCIL) { 11030b57cec5SDimitry Andric assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64"); 11040b57cec5SDimitry Andric 1105480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 1106480093f4SDimitry Andric 11070b57cec5SDimitry Andric // We want to emit the following pattern, which follows the x86 calling 11080b57cec5SDimitry Andric // convention to prepare for the trampoline call to be patched in. 11090b57cec5SDimitry Andric // 11100b57cec5SDimitry Andric // .p2align 1, ... 11110b57cec5SDimitry Andric // .Lxray_event_sled_N: 11120b57cec5SDimitry Andric // jmp +N // jump across the instrumentation sled 11130b57cec5SDimitry Andric // ... // set up arguments in register 11140b57cec5SDimitry Andric // callq __xray_CustomEvent@plt // force dependency to symbol 11150b57cec5SDimitry Andric // ... 11160b57cec5SDimitry Andric // <jump here> 11170b57cec5SDimitry Andric // 11180b57cec5SDimitry Andric // After patching, it would look something like: 11190b57cec5SDimitry Andric // 11200b57cec5SDimitry Andric // nopw (2-byte nop) 11210b57cec5SDimitry Andric // ... 11220b57cec5SDimitry Andric // callq __xrayCustomEvent // already lowered 11230b57cec5SDimitry Andric // ... 11240b57cec5SDimitry Andric // 11250b57cec5SDimitry Andric // --- 11260b57cec5SDimitry Andric // First we emit the label and the jump. 11270b57cec5SDimitry Andric auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true); 11280b57cec5SDimitry Andric OutStreamer->AddComment("# XRay Custom Event Log"); 1129bdd1243dSDimitry Andric OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo()); 11305ffd83dbSDimitry Andric OutStreamer->emitLabel(CurSled); 11310b57cec5SDimitry Andric 11320b57cec5SDimitry Andric // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as 11330b57cec5SDimitry Andric // an operand (computed as an offset from the jmp instruction). 11340b57cec5SDimitry Andric // FIXME: Find another less hacky way do force the relative jump. 11355ffd83dbSDimitry Andric OutStreamer->emitBinaryData("\xeb\x0f"); 11360b57cec5SDimitry Andric 11370b57cec5SDimitry Andric // The default C calling convention will place two arguments into %rcx and 11380b57cec5SDimitry Andric // %rdx -- so we only work with those. 1139480093f4SDimitry Andric const Register DestRegs[] = {X86::RDI, X86::RSI}; 11400b57cec5SDimitry Andric bool UsedMask[] = {false, false}; 11410b57cec5SDimitry Andric // Filled out in loop. 1142480093f4SDimitry Andric Register SrcRegs[] = {0, 0}; 11430b57cec5SDimitry Andric 11440b57cec5SDimitry Andric // Then we put the operands in the %rdi and %rsi registers. We spill the 11450b57cec5SDimitry Andric // values in the register before we clobber them, and mark them as used in 11460b57cec5SDimitry Andric // UsedMask. In case the arguments are already in the correct register, we use 11470b57cec5SDimitry Andric // emit nops appropriately sized to keep the sled the same size in every 11480b57cec5SDimitry Andric // situation. 11490b57cec5SDimitry Andric for (unsigned I = 0; I < MI.getNumOperands(); ++I) 1150*0fca6ea1SDimitry Andric if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I)); 1151*0fca6ea1SDimitry Andric Op.isValid()) { 1152*0fca6ea1SDimitry Andric assert(Op.isReg() && "Only support arguments in registers"); 1153*0fca6ea1SDimitry Andric SrcRegs[I] = getX86SubSuperRegister(Op.getReg(), 64); 115406c3fb27SDimitry Andric assert(SrcRegs[I].isValid() && "Invalid operand"); 11550b57cec5SDimitry Andric if (SrcRegs[I] != DestRegs[I]) { 11560b57cec5SDimitry Andric UsedMask[I] = true; 11570b57cec5SDimitry Andric EmitAndCountInstruction( 11580b57cec5SDimitry Andric MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I])); 11590b57cec5SDimitry Andric } else { 11605ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 4, Subtarget); 11610b57cec5SDimitry Andric } 11620b57cec5SDimitry Andric } 11630b57cec5SDimitry Andric 11640b57cec5SDimitry Andric // Now that the register values are stashed, mov arguments into place. 1165480093f4SDimitry Andric // FIXME: This doesn't work if one of the later SrcRegs is equal to an 1166480093f4SDimitry Andric // earlier DestReg. We will have already overwritten over the register before 1167480093f4SDimitry Andric // we can copy from it. 11680b57cec5SDimitry Andric for (unsigned I = 0; I < MI.getNumOperands(); ++I) 11690b57cec5SDimitry Andric if (SrcRegs[I] != DestRegs[I]) 11700b57cec5SDimitry Andric EmitAndCountInstruction( 11710b57cec5SDimitry Andric MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I])); 11720b57cec5SDimitry Andric 11730b57cec5SDimitry Andric // We emit a hard dependency on the __xray_CustomEvent symbol, which is the 11740b57cec5SDimitry Andric // name of the trampoline to be implemented by the XRay runtime. 11750b57cec5SDimitry Andric auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent"); 11760b57cec5SDimitry Andric MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym); 11770b57cec5SDimitry Andric if (isPositionIndependent()) 11780b57cec5SDimitry Andric TOp.setTargetFlags(X86II::MO_PLT); 11790b57cec5SDimitry Andric 11800b57cec5SDimitry Andric // Emit the call instruction. 11810b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32) 11820b57cec5SDimitry Andric .addOperand(MCIL.LowerSymbolOperand(TOp, TSym))); 11830b57cec5SDimitry Andric 11840b57cec5SDimitry Andric // Restore caller-saved and used registers. 11850b57cec5SDimitry Andric for (unsigned I = sizeof UsedMask; I-- > 0;) 11860b57cec5SDimitry Andric if (UsedMask[I]) 11870b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I])); 11880b57cec5SDimitry Andric else 11895ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 1, Subtarget); 11900b57cec5SDimitry Andric 11910b57cec5SDimitry Andric OutStreamer->AddComment("xray custom event end."); 11920b57cec5SDimitry Andric 11935ffd83dbSDimitry Andric // Record the sled version. Version 0 of this sled was spelled differently, so 11945ffd83dbSDimitry Andric // we let the runtime handle the different offsets we're using. Version 2 11955ffd83dbSDimitry Andric // changed the absolute address to a PC-relative address. 11965ffd83dbSDimitry Andric recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2); 11970b57cec5SDimitry Andric } 11980b57cec5SDimitry Andric 11990b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI, 12000b57cec5SDimitry Andric X86MCInstLower &MCIL) { 12010b57cec5SDimitry Andric assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64"); 12020b57cec5SDimitry Andric 1203480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 1204480093f4SDimitry Andric 12050b57cec5SDimitry Andric // We want to emit the following pattern, which follows the x86 calling 12060b57cec5SDimitry Andric // convention to prepare for the trampoline call to be patched in. 12070b57cec5SDimitry Andric // 12080b57cec5SDimitry Andric // .p2align 1, ... 12090b57cec5SDimitry Andric // .Lxray_event_sled_N: 12100b57cec5SDimitry Andric // jmp +N // jump across the instrumentation sled 12110b57cec5SDimitry Andric // ... // set up arguments in register 12120b57cec5SDimitry Andric // callq __xray_TypedEvent@plt // force dependency to symbol 12130b57cec5SDimitry Andric // ... 12140b57cec5SDimitry Andric // <jump here> 12150b57cec5SDimitry Andric // 12160b57cec5SDimitry Andric // After patching, it would look something like: 12170b57cec5SDimitry Andric // 12180b57cec5SDimitry Andric // nopw (2-byte nop) 12190b57cec5SDimitry Andric // ... 12200b57cec5SDimitry Andric // callq __xrayTypedEvent // already lowered 12210b57cec5SDimitry Andric // ... 12220b57cec5SDimitry Andric // 12230b57cec5SDimitry Andric // --- 12240b57cec5SDimitry Andric // First we emit the label and the jump. 12250b57cec5SDimitry Andric auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true); 12260b57cec5SDimitry Andric OutStreamer->AddComment("# XRay Typed Event Log"); 1227bdd1243dSDimitry Andric OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo()); 12285ffd83dbSDimitry Andric OutStreamer->emitLabel(CurSled); 12290b57cec5SDimitry Andric 12300b57cec5SDimitry Andric // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as 12310b57cec5SDimitry Andric // an operand (computed as an offset from the jmp instruction). 12320b57cec5SDimitry Andric // FIXME: Find another less hacky way do force the relative jump. 12335ffd83dbSDimitry Andric OutStreamer->emitBinaryData("\xeb\x14"); 12340b57cec5SDimitry Andric 12350b57cec5SDimitry Andric // An x86-64 convention may place three arguments into %rcx, %rdx, and R8, 12360b57cec5SDimitry Andric // so we'll work with those. Or we may be called via SystemV, in which case 12370b57cec5SDimitry Andric // we don't have to do any translation. 1238480093f4SDimitry Andric const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX}; 12390b57cec5SDimitry Andric bool UsedMask[] = {false, false, false}; 12400b57cec5SDimitry Andric 12410b57cec5SDimitry Andric // Will fill out src regs in the loop. 1242480093f4SDimitry Andric Register SrcRegs[] = {0, 0, 0}; 12430b57cec5SDimitry Andric 12440b57cec5SDimitry Andric // Then we put the operands in the SystemV registers. We spill the values in 12450b57cec5SDimitry Andric // the registers before we clobber them, and mark them as used in UsedMask. 12460b57cec5SDimitry Andric // In case the arguments are already in the correct register, we emit nops 12470b57cec5SDimitry Andric // appropriately sized to keep the sled the same size in every situation. 12480b57cec5SDimitry Andric for (unsigned I = 0; I < MI.getNumOperands(); ++I) 1249*0fca6ea1SDimitry Andric if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I)); 1250*0fca6ea1SDimitry Andric Op.isValid()) { 12510b57cec5SDimitry Andric // TODO: Is register only support adequate? 1252*0fca6ea1SDimitry Andric assert(Op.isReg() && "Only supports arguments in registers"); 1253*0fca6ea1SDimitry Andric SrcRegs[I] = getX86SubSuperRegister(Op.getReg(), 64); 125406c3fb27SDimitry Andric assert(SrcRegs[I].isValid() && "Invalid operand"); 12550b57cec5SDimitry Andric if (SrcRegs[I] != DestRegs[I]) { 12560b57cec5SDimitry Andric UsedMask[I] = true; 12570b57cec5SDimitry Andric EmitAndCountInstruction( 12580b57cec5SDimitry Andric MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I])); 12590b57cec5SDimitry Andric } else { 12605ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 4, Subtarget); 12610b57cec5SDimitry Andric } 12620b57cec5SDimitry Andric } 12630b57cec5SDimitry Andric 12640b57cec5SDimitry Andric // In the above loop we only stash all of the destination registers or emit 12650b57cec5SDimitry Andric // nops if the arguments are already in the right place. Doing the actually 12660b57cec5SDimitry Andric // moving is postponed until after all the registers are stashed so nothing 12670b57cec5SDimitry Andric // is clobbers. We've already added nops to account for the size of mov and 12680b57cec5SDimitry Andric // push if the register is in the right place, so we only have to worry about 12690b57cec5SDimitry Andric // emitting movs. 1270480093f4SDimitry Andric // FIXME: This doesn't work if one of the later SrcRegs is equal to an 1271480093f4SDimitry Andric // earlier DestReg. We will have already overwritten over the register before 1272480093f4SDimitry Andric // we can copy from it. 12730b57cec5SDimitry Andric for (unsigned I = 0; I < MI.getNumOperands(); ++I) 12740b57cec5SDimitry Andric if (UsedMask[I]) 12750b57cec5SDimitry Andric EmitAndCountInstruction( 12760b57cec5SDimitry Andric MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I])); 12770b57cec5SDimitry Andric 12780b57cec5SDimitry Andric // We emit a hard dependency on the __xray_TypedEvent symbol, which is the 12790b57cec5SDimitry Andric // name of the trampoline to be implemented by the XRay runtime. 12800b57cec5SDimitry Andric auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent"); 12810b57cec5SDimitry Andric MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym); 12820b57cec5SDimitry Andric if (isPositionIndependent()) 12830b57cec5SDimitry Andric TOp.setTargetFlags(X86II::MO_PLT); 12840b57cec5SDimitry Andric 12850b57cec5SDimitry Andric // Emit the call instruction. 12860b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32) 12870b57cec5SDimitry Andric .addOperand(MCIL.LowerSymbolOperand(TOp, TSym))); 12880b57cec5SDimitry Andric 12890b57cec5SDimitry Andric // Restore caller-saved and used registers. 12900b57cec5SDimitry Andric for (unsigned I = sizeof UsedMask; I-- > 0;) 12910b57cec5SDimitry Andric if (UsedMask[I]) 12920b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I])); 12930b57cec5SDimitry Andric else 12945ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 1, Subtarget); 12950b57cec5SDimitry Andric 12960b57cec5SDimitry Andric OutStreamer->AddComment("xray typed event end."); 12970b57cec5SDimitry Andric 12980b57cec5SDimitry Andric // Record the sled version. 12995ffd83dbSDimitry Andric recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2); 13000b57cec5SDimitry Andric } 13010b57cec5SDimitry Andric 13020b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI, 13030b57cec5SDimitry Andric X86MCInstLower &MCIL) { 1304480093f4SDimitry Andric 1305480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 1306480093f4SDimitry Andric 1307480093f4SDimitry Andric const Function &F = MF->getFunction(); 1308480093f4SDimitry Andric if (F.hasFnAttribute("patchable-function-entry")) { 1309480093f4SDimitry Andric unsigned Num; 1310480093f4SDimitry Andric if (F.getFnAttribute("patchable-function-entry") 1311480093f4SDimitry Andric .getValueAsString() 1312480093f4SDimitry Andric .getAsInteger(10, Num)) 1313480093f4SDimitry Andric return; 13145ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, Num, Subtarget); 1315480093f4SDimitry Andric return; 1316480093f4SDimitry Andric } 13170b57cec5SDimitry Andric // We want to emit the following pattern: 13180b57cec5SDimitry Andric // 13190b57cec5SDimitry Andric // .p2align 1, ... 13200b57cec5SDimitry Andric // .Lxray_sled_N: 13210b57cec5SDimitry Andric // jmp .tmpN 13220b57cec5SDimitry Andric // # 9 bytes worth of noops 13230b57cec5SDimitry Andric // 13240b57cec5SDimitry Andric // We need the 9 bytes because at runtime, we'd be patching over the full 11 13250b57cec5SDimitry Andric // bytes with the following pattern: 13260b57cec5SDimitry Andric // 13270b57cec5SDimitry Andric // mov %r10, <function id, 32-bit> // 6 bytes 13280b57cec5SDimitry Andric // call <relative offset, 32-bits> // 5 bytes 13290b57cec5SDimitry Andric // 13300b57cec5SDimitry Andric auto CurSled = OutContext.createTempSymbol("xray_sled_", true); 1331bdd1243dSDimitry Andric OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo()); 13325ffd83dbSDimitry Andric OutStreamer->emitLabel(CurSled); 13330b57cec5SDimitry Andric 13340b57cec5SDimitry Andric // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as 13350b57cec5SDimitry Andric // an operand (computed as an offset from the jmp instruction). 13360b57cec5SDimitry Andric // FIXME: Find another less hacky way do force the relative jump. 13375ffd83dbSDimitry Andric OutStreamer->emitBytes("\xeb\x09"); 13385ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 9, Subtarget); 13395ffd83dbSDimitry Andric recordSled(CurSled, MI, SledKind::FUNCTION_ENTER, 2); 13400b57cec5SDimitry Andric } 13410b57cec5SDimitry Andric 13420b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI, 13430b57cec5SDimitry Andric X86MCInstLower &MCIL) { 1344480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 1345480093f4SDimitry Andric 13460b57cec5SDimitry Andric // Since PATCHABLE_RET takes the opcode of the return statement as an 13470b57cec5SDimitry Andric // argument, we use that to emit the correct form of the RET that we want. 13480b57cec5SDimitry Andric // i.e. when we see this: 13490b57cec5SDimitry Andric // 13500b57cec5SDimitry Andric // PATCHABLE_RET X86::RET ... 13510b57cec5SDimitry Andric // 13520b57cec5SDimitry Andric // We should emit the RET followed by sleds. 13530b57cec5SDimitry Andric // 13540b57cec5SDimitry Andric // .p2align 1, ... 13550b57cec5SDimitry Andric // .Lxray_sled_N: 13560b57cec5SDimitry Andric // ret # or equivalent instruction 13570b57cec5SDimitry Andric // # 10 bytes worth of noops 13580b57cec5SDimitry Andric // 13590b57cec5SDimitry Andric // This just makes sure that the alignment for the next instruction is 2. 13600b57cec5SDimitry Andric auto CurSled = OutContext.createTempSymbol("xray_sled_", true); 1361bdd1243dSDimitry Andric OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo()); 13625ffd83dbSDimitry Andric OutStreamer->emitLabel(CurSled); 13630b57cec5SDimitry Andric unsigned OpCode = MI.getOperand(0).getImm(); 13640b57cec5SDimitry Andric MCInst Ret; 13650b57cec5SDimitry Andric Ret.setOpcode(OpCode); 1366e8d8bef9SDimitry Andric for (auto &MO : drop_begin(MI.operands())) 1367*0fca6ea1SDimitry Andric if (auto Op = MCIL.LowerMachineOperand(&MI, MO); Op.isValid()) 1368*0fca6ea1SDimitry Andric Ret.addOperand(Op); 13695ffd83dbSDimitry Andric OutStreamer->emitInstruction(Ret, getSubtargetInfo()); 13705ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 10, Subtarget); 13715ffd83dbSDimitry Andric recordSled(CurSled, MI, SledKind::FUNCTION_EXIT, 2); 13720b57cec5SDimitry Andric } 13730b57cec5SDimitry Andric 13740b57cec5SDimitry Andric void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI, 13750b57cec5SDimitry Andric X86MCInstLower &MCIL) { 1376*0fca6ea1SDimitry Andric MCInst TC; 1377*0fca6ea1SDimitry Andric TC.setOpcode(convertTailJumpOpcode(MI.getOperand(0).getImm())); 1378*0fca6ea1SDimitry Andric // Drop the tail jump opcode. 1379*0fca6ea1SDimitry Andric auto TCOperands = drop_begin(MI.operands()); 1380*0fca6ea1SDimitry Andric bool IsConditional = TC.getOpcode() == X86::JCC_1; 1381*0fca6ea1SDimitry Andric MCSymbol *FallthroughLabel; 1382*0fca6ea1SDimitry Andric if (IsConditional) { 1383*0fca6ea1SDimitry Andric // Rewrite: 1384*0fca6ea1SDimitry Andric // je target 1385*0fca6ea1SDimitry Andric // 1386*0fca6ea1SDimitry Andric // To: 1387*0fca6ea1SDimitry Andric // jne .fallthrough 1388*0fca6ea1SDimitry Andric // .p2align 1, ... 1389*0fca6ea1SDimitry Andric // .Lxray_sled_N: 1390*0fca6ea1SDimitry Andric // SLED_CODE 1391*0fca6ea1SDimitry Andric // jmp target 1392*0fca6ea1SDimitry Andric // .fallthrough: 1393*0fca6ea1SDimitry Andric FallthroughLabel = OutContext.createTempSymbol(); 1394*0fca6ea1SDimitry Andric EmitToStreamer( 1395*0fca6ea1SDimitry Andric *OutStreamer, 1396*0fca6ea1SDimitry Andric MCInstBuilder(X86::JCC_1) 1397*0fca6ea1SDimitry Andric .addExpr(MCSymbolRefExpr::create(FallthroughLabel, OutContext)) 1398*0fca6ea1SDimitry Andric .addImm(X86::GetOppositeBranchCondition( 1399*0fca6ea1SDimitry Andric static_cast<X86::CondCode>(MI.getOperand(2).getImm())))); 1400*0fca6ea1SDimitry Andric TC.setOpcode(X86::JMP_1); 1401*0fca6ea1SDimitry Andric // Drop the condition code. 1402*0fca6ea1SDimitry Andric TCOperands = drop_end(TCOperands); 1403*0fca6ea1SDimitry Andric } 1404*0fca6ea1SDimitry Andric 1405480093f4SDimitry Andric NoAutoPaddingScope NoPadScope(*OutStreamer); 1406480093f4SDimitry Andric 14070b57cec5SDimitry Andric // Like PATCHABLE_RET, we have the actual instruction in the operands to this 14080b57cec5SDimitry Andric // instruction so we lower that particular instruction and its operands. 14090b57cec5SDimitry Andric // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how 14100b57cec5SDimitry Andric // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to 14110b57cec5SDimitry Andric // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual 14120b57cec5SDimitry Andric // tail call much like how we have it in PATCHABLE_RET. 14130b57cec5SDimitry Andric auto CurSled = OutContext.createTempSymbol("xray_sled_", true); 1414bdd1243dSDimitry Andric OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo()); 14155ffd83dbSDimitry Andric OutStreamer->emitLabel(CurSled); 14160b57cec5SDimitry Andric auto Target = OutContext.createTempSymbol(); 14170b57cec5SDimitry Andric 14180b57cec5SDimitry Andric // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as 14190b57cec5SDimitry Andric // an operand (computed as an offset from the jmp instruction). 14200b57cec5SDimitry Andric // FIXME: Find another less hacky way do force the relative jump. 14215ffd83dbSDimitry Andric OutStreamer->emitBytes("\xeb\x09"); 14225ffd83dbSDimitry Andric emitX86Nops(*OutStreamer, 9, Subtarget); 14235ffd83dbSDimitry Andric OutStreamer->emitLabel(Target); 14245ffd83dbSDimitry Andric recordSled(CurSled, MI, SledKind::TAIL_CALL, 2); 14250b57cec5SDimitry Andric 14260b57cec5SDimitry Andric // Before emitting the instruction, add a comment to indicate that this is 14270b57cec5SDimitry Andric // indeed a tail call. 14280b57cec5SDimitry Andric OutStreamer->AddComment("TAILCALL"); 1429*0fca6ea1SDimitry Andric for (auto &MO : TCOperands) 1430*0fca6ea1SDimitry Andric if (auto Op = MCIL.LowerMachineOperand(&MI, MO); Op.isValid()) 1431*0fca6ea1SDimitry Andric TC.addOperand(Op); 14325ffd83dbSDimitry Andric OutStreamer->emitInstruction(TC, getSubtargetInfo()); 1433*0fca6ea1SDimitry Andric 1434*0fca6ea1SDimitry Andric if (IsConditional) 1435*0fca6ea1SDimitry Andric OutStreamer->emitLabel(FallthroughLabel); 14360b57cec5SDimitry Andric } 14370b57cec5SDimitry Andric 14380b57cec5SDimitry Andric // Returns instruction preceding MBBI in MachineFunction. 14390b57cec5SDimitry Andric // If MBBI is the first instruction of the first basic block, returns null. 14400b57cec5SDimitry Andric static MachineBasicBlock::const_iterator 14410b57cec5SDimitry Andric PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI) { 14420b57cec5SDimitry Andric const MachineBasicBlock *MBB = MBBI->getParent(); 14430b57cec5SDimitry Andric while (MBBI == MBB->begin()) { 14440b57cec5SDimitry Andric if (MBB == &MBB->getParent()->front()) 14450b57cec5SDimitry Andric return MachineBasicBlock::const_iterator(); 14460b57cec5SDimitry Andric MBB = MBB->getPrevNode(); 14470b57cec5SDimitry Andric MBBI = MBB->end(); 14480b57cec5SDimitry Andric } 14490b57cec5SDimitry Andric --MBBI; 14500b57cec5SDimitry Andric return MBBI; 14510b57cec5SDimitry Andric } 14520b57cec5SDimitry Andric 1453*0fca6ea1SDimitry Andric static unsigned getSrcIdx(const MachineInstr* MI, unsigned SrcIdx) { 1454*0fca6ea1SDimitry Andric if (X86II::isKMasked(MI->getDesc().TSFlags)) { 1455*0fca6ea1SDimitry Andric // Skip mask operand. 1456*0fca6ea1SDimitry Andric ++SrcIdx; 1457*0fca6ea1SDimitry Andric if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) { 1458*0fca6ea1SDimitry Andric // Skip passthru operand. 1459*0fca6ea1SDimitry Andric ++SrcIdx; 1460*0fca6ea1SDimitry Andric } 1461*0fca6ea1SDimitry Andric } 1462*0fca6ea1SDimitry Andric return SrcIdx; 1463*0fca6ea1SDimitry Andric } 14640b57cec5SDimitry Andric 1465*0fca6ea1SDimitry Andric static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI, 1466*0fca6ea1SDimitry Andric unsigned SrcOpIdx) { 14670b57cec5SDimitry Andric const MachineOperand &DstOp = MI->getOperand(0); 1468*0fca6ea1SDimitry Andric CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()); 14690b57cec5SDimitry Andric 1470*0fca6ea1SDimitry Andric // Handle AVX512 MASK/MASXZ write mask comments. 1471*0fca6ea1SDimitry Andric // MASK: zmmX {%kY} 1472*0fca6ea1SDimitry Andric // MASKZ: zmmX {%kY} {z} 1473*0fca6ea1SDimitry Andric if (X86II::isKMasked(MI->getDesc().TSFlags)) { 1474*0fca6ea1SDimitry Andric const MachineOperand &WriteMaskOp = MI->getOperand(SrcOpIdx - 1); 1475*0fca6ea1SDimitry Andric StringRef Mask = X86ATTInstPrinter::getRegisterName(WriteMaskOp.getReg()); 1476*0fca6ea1SDimitry Andric CS << " {%" << Mask << "}"; 1477*0fca6ea1SDimitry Andric if (!X86II::isKMergeMasked(MI->getDesc().TSFlags)) { 1478*0fca6ea1SDimitry Andric CS << " {z}"; 1479*0fca6ea1SDimitry Andric } 1480*0fca6ea1SDimitry Andric } 1481*0fca6ea1SDimitry Andric } 14820b57cec5SDimitry Andric 1483*0fca6ea1SDimitry Andric static void printShuffleMask(raw_ostream &CS, StringRef Src1Name, 1484*0fca6ea1SDimitry Andric StringRef Src2Name, ArrayRef<int> Mask) { 14850b57cec5SDimitry Andric // One source operand, fix the mask to print all elements in one span. 1486bdd1243dSDimitry Andric SmallVector<int, 8> ShuffleMask(Mask); 14870b57cec5SDimitry Andric if (Src1Name == Src2Name) 14880b57cec5SDimitry Andric for (int i = 0, e = ShuffleMask.size(); i != e; ++i) 14890b57cec5SDimitry Andric if (ShuffleMask[i] >= e) 14900b57cec5SDimitry Andric ShuffleMask[i] -= e; 14910b57cec5SDimitry Andric 14920b57cec5SDimitry Andric for (int i = 0, e = ShuffleMask.size(); i != e; ++i) { 14930b57cec5SDimitry Andric if (i != 0) 14940b57cec5SDimitry Andric CS << ","; 14950b57cec5SDimitry Andric if (ShuffleMask[i] == SM_SentinelZero) { 14960b57cec5SDimitry Andric CS << "zero"; 14970b57cec5SDimitry Andric continue; 14980b57cec5SDimitry Andric } 14990b57cec5SDimitry Andric 15000b57cec5SDimitry Andric // Otherwise, it must come from src1 or src2. Print the span of elements 15010b57cec5SDimitry Andric // that comes from this src. 15020b57cec5SDimitry Andric bool isSrc1 = ShuffleMask[i] < (int)e; 15030b57cec5SDimitry Andric CS << (isSrc1 ? Src1Name : Src2Name) << '['; 15040b57cec5SDimitry Andric 15050b57cec5SDimitry Andric bool IsFirst = true; 15060b57cec5SDimitry Andric while (i != e && ShuffleMask[i] != SM_SentinelZero && 15070b57cec5SDimitry Andric (ShuffleMask[i] < (int)e) == isSrc1) { 15080b57cec5SDimitry Andric if (!IsFirst) 15090b57cec5SDimitry Andric CS << ','; 15100b57cec5SDimitry Andric else 15110b57cec5SDimitry Andric IsFirst = false; 15120b57cec5SDimitry Andric if (ShuffleMask[i] == SM_SentinelUndef) 15130b57cec5SDimitry Andric CS << "u"; 15140b57cec5SDimitry Andric else 15150b57cec5SDimitry Andric CS << ShuffleMask[i] % (int)e; 15160b57cec5SDimitry Andric ++i; 15170b57cec5SDimitry Andric } 15180b57cec5SDimitry Andric CS << ']'; 15190b57cec5SDimitry Andric --i; // For loop increments element #. 15200b57cec5SDimitry Andric } 1521*0fca6ea1SDimitry Andric } 1522*0fca6ea1SDimitry Andric 1523*0fca6ea1SDimitry Andric static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx, 1524*0fca6ea1SDimitry Andric unsigned SrcOp2Idx, ArrayRef<int> Mask) { 1525*0fca6ea1SDimitry Andric std::string Comment; 1526*0fca6ea1SDimitry Andric 1527*0fca6ea1SDimitry Andric const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx); 1528*0fca6ea1SDimitry Andric const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx); 1529*0fca6ea1SDimitry Andric StringRef Src1Name = SrcOp1.isReg() 1530*0fca6ea1SDimitry Andric ? X86ATTInstPrinter::getRegisterName(SrcOp1.getReg()) 1531*0fca6ea1SDimitry Andric : "mem"; 1532*0fca6ea1SDimitry Andric StringRef Src2Name = SrcOp2.isReg() 1533*0fca6ea1SDimitry Andric ? X86ATTInstPrinter::getRegisterName(SrcOp2.getReg()) 1534*0fca6ea1SDimitry Andric : "mem"; 1535*0fca6ea1SDimitry Andric 1536*0fca6ea1SDimitry Andric raw_string_ostream CS(Comment); 1537*0fca6ea1SDimitry Andric printDstRegisterName(CS, MI, SrcOp1Idx); 1538*0fca6ea1SDimitry Andric CS << " = "; 1539*0fca6ea1SDimitry Andric printShuffleMask(CS, Src1Name, Src2Name, Mask); 15400b57cec5SDimitry Andric CS.flush(); 15410b57cec5SDimitry Andric 15420b57cec5SDimitry Andric return Comment; 15430b57cec5SDimitry Andric } 15440b57cec5SDimitry Andric 15457a6dacacSDimitry Andric static void printConstant(const APInt &Val, raw_ostream &CS, 15467a6dacacSDimitry Andric bool PrintZero = false) { 15470b57cec5SDimitry Andric if (Val.getBitWidth() <= 64) { 15487a6dacacSDimitry Andric CS << (PrintZero ? 0ULL : Val.getZExtValue()); 15490b57cec5SDimitry Andric } else { 15500b57cec5SDimitry Andric // print multi-word constant as (w0,w1) 15510b57cec5SDimitry Andric CS << "("; 15520b57cec5SDimitry Andric for (int i = 0, N = Val.getNumWords(); i < N; ++i) { 15530b57cec5SDimitry Andric if (i > 0) 15540b57cec5SDimitry Andric CS << ","; 15557a6dacacSDimitry Andric CS << (PrintZero ? 0ULL : Val.getRawData()[i]); 15560b57cec5SDimitry Andric } 15570b57cec5SDimitry Andric CS << ")"; 15580b57cec5SDimitry Andric } 15590b57cec5SDimitry Andric } 15600b57cec5SDimitry Andric 15617a6dacacSDimitry Andric static void printConstant(const APFloat &Flt, raw_ostream &CS, 15627a6dacacSDimitry Andric bool PrintZero = false) { 15630b57cec5SDimitry Andric SmallString<32> Str; 15647a6dacacSDimitry Andric // Force scientific notation to distinguish from integers. 15657a6dacacSDimitry Andric if (PrintZero) 15667a6dacacSDimitry Andric APFloat::getZero(Flt.getSemantics()).toString(Str, 0, 0); 15677a6dacacSDimitry Andric else 15680b57cec5SDimitry Andric Flt.toString(Str, 0, 0); 15690b57cec5SDimitry Andric CS << Str; 15700b57cec5SDimitry Andric } 15710b57cec5SDimitry Andric 157206c3fb27SDimitry Andric static void printConstant(const Constant *COp, unsigned BitWidth, 15737a6dacacSDimitry Andric raw_ostream &CS, bool PrintZero = false) { 15740b57cec5SDimitry Andric if (isa<UndefValue>(COp)) { 15750b57cec5SDimitry Andric CS << "u"; 15760b57cec5SDimitry Andric } else if (auto *CI = dyn_cast<ConstantInt>(COp)) { 15777a6dacacSDimitry Andric printConstant(CI->getValue(), CS, PrintZero); 15780b57cec5SDimitry Andric } else if (auto *CF = dyn_cast<ConstantFP>(COp)) { 15797a6dacacSDimitry Andric printConstant(CF->getValueAPF(), CS, PrintZero); 158006c3fb27SDimitry Andric } else if (auto *CDS = dyn_cast<ConstantDataSequential>(COp)) { 158106c3fb27SDimitry Andric Type *EltTy = CDS->getElementType(); 158206c3fb27SDimitry Andric bool IsInteger = EltTy->isIntegerTy(); 158306c3fb27SDimitry Andric bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy(); 158406c3fb27SDimitry Andric unsigned EltBits = EltTy->getPrimitiveSizeInBits(); 158506c3fb27SDimitry Andric unsigned E = std::min(BitWidth / EltBits, CDS->getNumElements()); 15867a6dacacSDimitry Andric assert((BitWidth % EltBits) == 0 && "Element size mismatch"); 158706c3fb27SDimitry Andric for (unsigned I = 0; I != E; ++I) { 158806c3fb27SDimitry Andric if (I != 0) 158906c3fb27SDimitry Andric CS << ","; 159006c3fb27SDimitry Andric if (IsInteger) 15917a6dacacSDimitry Andric printConstant(CDS->getElementAsAPInt(I), CS, PrintZero); 159206c3fb27SDimitry Andric else if (IsFP) 15937a6dacacSDimitry Andric printConstant(CDS->getElementAsAPFloat(I), CS, PrintZero); 159406c3fb27SDimitry Andric else 159506c3fb27SDimitry Andric CS << "?"; 159606c3fb27SDimitry Andric } 15977a6dacacSDimitry Andric } else if (auto *CV = dyn_cast<ConstantVector>(COp)) { 15987a6dacacSDimitry Andric unsigned EltBits = CV->getType()->getScalarSizeInBits(); 15997a6dacacSDimitry Andric unsigned E = std::min(BitWidth / EltBits, CV->getNumOperands()); 16007a6dacacSDimitry Andric assert((BitWidth % EltBits) == 0 && "Element size mismatch"); 16017a6dacacSDimitry Andric for (unsigned I = 0; I != E; ++I) { 16027a6dacacSDimitry Andric if (I != 0) 16037a6dacacSDimitry Andric CS << ","; 16047a6dacacSDimitry Andric printConstant(CV->getOperand(I), EltBits, CS, PrintZero); 16057a6dacacSDimitry Andric } 16060b57cec5SDimitry Andric } else { 16070b57cec5SDimitry Andric CS << "?"; 16080b57cec5SDimitry Andric } 16090b57cec5SDimitry Andric } 16100b57cec5SDimitry Andric 16117a6dacacSDimitry Andric static void printZeroUpperMove(const MachineInstr *MI, MCStreamer &OutStreamer, 16127a6dacacSDimitry Andric int SclWidth, int VecWidth, 16137a6dacacSDimitry Andric const char *ShuffleComment) { 1614*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 1615*0fca6ea1SDimitry Andric 16167a6dacacSDimitry Andric std::string Comment; 16177a6dacacSDimitry Andric raw_string_ostream CS(Comment); 1618*0fca6ea1SDimitry Andric printDstRegisterName(CS, MI, SrcIdx); 1619*0fca6ea1SDimitry Andric CS << " = "; 16207a6dacacSDimitry Andric 1621*0fca6ea1SDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx)) { 16227a6dacacSDimitry Andric CS << "["; 16237a6dacacSDimitry Andric printConstant(C, SclWidth, CS); 16247a6dacacSDimitry Andric for (int I = 1, E = VecWidth / SclWidth; I < E; ++I) { 16257a6dacacSDimitry Andric CS << ","; 16267a6dacacSDimitry Andric printConstant(C, SclWidth, CS, true); 16277a6dacacSDimitry Andric } 16287a6dacacSDimitry Andric CS << "]"; 16297a6dacacSDimitry Andric OutStreamer.AddComment(CS.str()); 16307a6dacacSDimitry Andric return; // early-out 16317a6dacacSDimitry Andric } 16327a6dacacSDimitry Andric 16337a6dacacSDimitry Andric // We didn't find a constant load, fallback to a shuffle mask decode. 16347a6dacacSDimitry Andric CS << ShuffleComment; 16357a6dacacSDimitry Andric OutStreamer.AddComment(CS.str()); 16367a6dacacSDimitry Andric } 16377a6dacacSDimitry Andric 16387a6dacacSDimitry Andric static void printBroadcast(const MachineInstr *MI, MCStreamer &OutStreamer, 16397a6dacacSDimitry Andric int Repeats, int BitWidth) { 1640*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 1641*0fca6ea1SDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx)) { 16427a6dacacSDimitry Andric std::string Comment; 16437a6dacacSDimitry Andric raw_string_ostream CS(Comment); 1644*0fca6ea1SDimitry Andric printDstRegisterName(CS, MI, SrcIdx); 1645*0fca6ea1SDimitry Andric CS << " = ["; 16467a6dacacSDimitry Andric for (int l = 0; l != Repeats; ++l) { 16477a6dacacSDimitry Andric if (l != 0) 16487a6dacacSDimitry Andric CS << ","; 16497a6dacacSDimitry Andric printConstant(C, BitWidth, CS); 16507a6dacacSDimitry Andric } 16517a6dacacSDimitry Andric CS << "]"; 16527a6dacacSDimitry Andric OutStreamer.AddComment(CS.str()); 16537a6dacacSDimitry Andric } 16547a6dacacSDimitry Andric } 16557a6dacacSDimitry Andric 1656*0fca6ea1SDimitry Andric static bool printExtend(const MachineInstr *MI, MCStreamer &OutStreamer, 1657*0fca6ea1SDimitry Andric int SrcEltBits, int DstEltBits, bool IsSext) { 1658*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 1659*0fca6ea1SDimitry Andric auto *C = X86::getConstantFromPool(*MI, SrcIdx); 1660*0fca6ea1SDimitry Andric if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) { 1661*0fca6ea1SDimitry Andric if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) { 1662*0fca6ea1SDimitry Andric int NumElts = CDS->getNumElements(); 1663*0fca6ea1SDimitry Andric std::string Comment; 1664*0fca6ea1SDimitry Andric raw_string_ostream CS(Comment); 1665*0fca6ea1SDimitry Andric printDstRegisterName(CS, MI, SrcIdx); 1666*0fca6ea1SDimitry Andric CS << " = ["; 1667*0fca6ea1SDimitry Andric for (int i = 0; i != NumElts; ++i) { 1668*0fca6ea1SDimitry Andric if (i != 0) 1669*0fca6ea1SDimitry Andric CS << ","; 1670*0fca6ea1SDimitry Andric if (CDS->getElementType()->isIntegerTy()) { 1671*0fca6ea1SDimitry Andric APInt Elt = CDS->getElementAsAPInt(i); 1672*0fca6ea1SDimitry Andric Elt = IsSext ? Elt.sext(DstEltBits) : Elt.zext(DstEltBits); 1673*0fca6ea1SDimitry Andric printConstant(Elt, CS); 1674*0fca6ea1SDimitry Andric } else 1675*0fca6ea1SDimitry Andric CS << "?"; 1676*0fca6ea1SDimitry Andric } 1677*0fca6ea1SDimitry Andric CS << "]"; 1678*0fca6ea1SDimitry Andric OutStreamer.AddComment(CS.str()); 1679*0fca6ea1SDimitry Andric return true; 1680*0fca6ea1SDimitry Andric } 1681*0fca6ea1SDimitry Andric } 1682*0fca6ea1SDimitry Andric 1683*0fca6ea1SDimitry Andric return false; 1684*0fca6ea1SDimitry Andric } 1685*0fca6ea1SDimitry Andric static void printSignExtend(const MachineInstr *MI, MCStreamer &OutStreamer, 1686*0fca6ea1SDimitry Andric int SrcEltBits, int DstEltBits) { 1687*0fca6ea1SDimitry Andric printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, true); 1688*0fca6ea1SDimitry Andric } 1689*0fca6ea1SDimitry Andric static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer, 1690*0fca6ea1SDimitry Andric int SrcEltBits, int DstEltBits) { 1691*0fca6ea1SDimitry Andric if (printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, false)) 1692*0fca6ea1SDimitry Andric return; 1693*0fca6ea1SDimitry Andric 1694*0fca6ea1SDimitry Andric // We didn't find a constant load, fallback to a shuffle mask decode. 1695*0fca6ea1SDimitry Andric std::string Comment; 1696*0fca6ea1SDimitry Andric raw_string_ostream CS(Comment); 1697*0fca6ea1SDimitry Andric printDstRegisterName(CS, MI, getSrcIdx(MI, 1)); 1698*0fca6ea1SDimitry Andric CS << " = "; 1699*0fca6ea1SDimitry Andric 1700*0fca6ea1SDimitry Andric SmallVector<int> Mask; 1701*0fca6ea1SDimitry Andric unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 1702*0fca6ea1SDimitry Andric assert((Width % DstEltBits) == 0 && (DstEltBits % SrcEltBits) == 0 && 1703*0fca6ea1SDimitry Andric "Illegal extension ratio"); 1704*0fca6ea1SDimitry Andric DecodeZeroExtendMask(SrcEltBits, DstEltBits, Width / DstEltBits, false, Mask); 1705*0fca6ea1SDimitry Andric printShuffleMask(CS, "mem", "", Mask); 1706*0fca6ea1SDimitry Andric 1707*0fca6ea1SDimitry Andric OutStreamer.AddComment(CS.str()); 1708*0fca6ea1SDimitry Andric } 1709*0fca6ea1SDimitry Andric 17100b57cec5SDimitry Andric void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) { 17110b57cec5SDimitry Andric assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); 171206c3fb27SDimitry Andric assert((getSubtarget().isOSWindows() || TM.getTargetTriple().isUEFI()) && 171306c3fb27SDimitry Andric "SEH_ instruction Windows and UEFI only"); 17140b57cec5SDimitry Andric 17150b57cec5SDimitry Andric // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86. 17160b57cec5SDimitry Andric if (EmitFPOData) { 17170b57cec5SDimitry Andric X86TargetStreamer *XTS = 17180b57cec5SDimitry Andric static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer()); 17190b57cec5SDimitry Andric switch (MI->getOpcode()) { 17200b57cec5SDimitry Andric case X86::SEH_PushReg: 17210b57cec5SDimitry Andric XTS->emitFPOPushReg(MI->getOperand(0).getImm()); 17220b57cec5SDimitry Andric break; 17230b57cec5SDimitry Andric case X86::SEH_StackAlloc: 17240b57cec5SDimitry Andric XTS->emitFPOStackAlloc(MI->getOperand(0).getImm()); 17250b57cec5SDimitry Andric break; 17260b57cec5SDimitry Andric case X86::SEH_StackAlign: 17270b57cec5SDimitry Andric XTS->emitFPOStackAlign(MI->getOperand(0).getImm()); 17280b57cec5SDimitry Andric break; 17290b57cec5SDimitry Andric case X86::SEH_SetFrame: 17300b57cec5SDimitry Andric assert(MI->getOperand(1).getImm() == 0 && 17310b57cec5SDimitry Andric ".cv_fpo_setframe takes no offset"); 17320b57cec5SDimitry Andric XTS->emitFPOSetFrame(MI->getOperand(0).getImm()); 17330b57cec5SDimitry Andric break; 17340b57cec5SDimitry Andric case X86::SEH_EndPrologue: 17350b57cec5SDimitry Andric XTS->emitFPOEndPrologue(); 17360b57cec5SDimitry Andric break; 17370b57cec5SDimitry Andric case X86::SEH_SaveReg: 17380b57cec5SDimitry Andric case X86::SEH_SaveXMM: 17390b57cec5SDimitry Andric case X86::SEH_PushFrame: 17400b57cec5SDimitry Andric llvm_unreachable("SEH_ directive incompatible with FPO"); 17410b57cec5SDimitry Andric break; 17420b57cec5SDimitry Andric default: 17430b57cec5SDimitry Andric llvm_unreachable("expected SEH_ instruction"); 17440b57cec5SDimitry Andric } 17450b57cec5SDimitry Andric return; 17460b57cec5SDimitry Andric } 17470b57cec5SDimitry Andric 17480b57cec5SDimitry Andric // Otherwise, use the .seh_ directives for all other Windows platforms. 17490b57cec5SDimitry Andric switch (MI->getOpcode()) { 17500b57cec5SDimitry Andric case X86::SEH_PushReg: 175181ad6265SDimitry Andric OutStreamer->emitWinCFIPushReg(MI->getOperand(0).getImm()); 17520b57cec5SDimitry Andric break; 17530b57cec5SDimitry Andric 17540b57cec5SDimitry Andric case X86::SEH_SaveReg: 175581ad6265SDimitry Andric OutStreamer->emitWinCFISaveReg(MI->getOperand(0).getImm(), 17560b57cec5SDimitry Andric MI->getOperand(1).getImm()); 17570b57cec5SDimitry Andric break; 17580b57cec5SDimitry Andric 17590b57cec5SDimitry Andric case X86::SEH_SaveXMM: 176081ad6265SDimitry Andric OutStreamer->emitWinCFISaveXMM(MI->getOperand(0).getImm(), 17610b57cec5SDimitry Andric MI->getOperand(1).getImm()); 17620b57cec5SDimitry Andric break; 17630b57cec5SDimitry Andric 17640b57cec5SDimitry Andric case X86::SEH_StackAlloc: 176581ad6265SDimitry Andric OutStreamer->emitWinCFIAllocStack(MI->getOperand(0).getImm()); 17660b57cec5SDimitry Andric break; 17670b57cec5SDimitry Andric 17680b57cec5SDimitry Andric case X86::SEH_SetFrame: 176981ad6265SDimitry Andric OutStreamer->emitWinCFISetFrame(MI->getOperand(0).getImm(), 17700b57cec5SDimitry Andric MI->getOperand(1).getImm()); 17710b57cec5SDimitry Andric break; 17720b57cec5SDimitry Andric 17730b57cec5SDimitry Andric case X86::SEH_PushFrame: 177481ad6265SDimitry Andric OutStreamer->emitWinCFIPushFrame(MI->getOperand(0).getImm()); 17750b57cec5SDimitry Andric break; 17760b57cec5SDimitry Andric 17770b57cec5SDimitry Andric case X86::SEH_EndPrologue: 177881ad6265SDimitry Andric OutStreamer->emitWinCFIEndProlog(); 17790b57cec5SDimitry Andric break; 17800b57cec5SDimitry Andric 17810b57cec5SDimitry Andric default: 17820b57cec5SDimitry Andric llvm_unreachable("expected SEH_ instruction"); 17830b57cec5SDimitry Andric } 17840b57cec5SDimitry Andric } 17850b57cec5SDimitry Andric 17865ffd83dbSDimitry Andric static void addConstantComments(const MachineInstr *MI, 17875ffd83dbSDimitry Andric MCStreamer &OutStreamer) { 17885ffd83dbSDimitry Andric switch (MI->getOpcode()) { 17895ffd83dbSDimitry Andric // Lower PSHUFB and VPERMILP normally but add a comment if we can find 17905ffd83dbSDimitry Andric // a constant shuffle mask. We won't be able to do this at the MC layer 17915ffd83dbSDimitry Andric // because the mask isn't an immediate. 17925ffd83dbSDimitry Andric case X86::PSHUFBrm: 17935ffd83dbSDimitry Andric case X86::VPSHUFBrm: 17945ffd83dbSDimitry Andric case X86::VPSHUFBYrm: 17955ffd83dbSDimitry Andric case X86::VPSHUFBZ128rm: 17965ffd83dbSDimitry Andric case X86::VPSHUFBZ128rmk: 17975ffd83dbSDimitry Andric case X86::VPSHUFBZ128rmkz: 17985ffd83dbSDimitry Andric case X86::VPSHUFBZ256rm: 17995ffd83dbSDimitry Andric case X86::VPSHUFBZ256rmk: 18005ffd83dbSDimitry Andric case X86::VPSHUFBZ256rmkz: 18015ffd83dbSDimitry Andric case X86::VPSHUFBZrm: 18025ffd83dbSDimitry Andric case X86::VPSHUFBZrmk: 18035ffd83dbSDimitry Andric case X86::VPSHUFBZrmkz: { 1804*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 18057a6dacacSDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) { 1806*0fca6ea1SDimitry Andric unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 18075ffd83dbSDimitry Andric SmallVector<int, 64> Mask; 18085ffd83dbSDimitry Andric DecodePSHUFBMask(C, Width, Mask); 18095ffd83dbSDimitry Andric if (!Mask.empty()) 18105ffd83dbSDimitry Andric OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); 18115ffd83dbSDimitry Andric } 18125ffd83dbSDimitry Andric break; 18135ffd83dbSDimitry Andric } 18145ffd83dbSDimitry Andric 18155ffd83dbSDimitry Andric case X86::VPERMILPSrm: 18165ffd83dbSDimitry Andric case X86::VPERMILPSYrm: 18175ffd83dbSDimitry Andric case X86::VPERMILPSZ128rm: 18185ffd83dbSDimitry Andric case X86::VPERMILPSZ128rmk: 18195ffd83dbSDimitry Andric case X86::VPERMILPSZ128rmkz: 18205ffd83dbSDimitry Andric case X86::VPERMILPSZ256rm: 18215ffd83dbSDimitry Andric case X86::VPERMILPSZ256rmk: 18225ffd83dbSDimitry Andric case X86::VPERMILPSZ256rmkz: 18235ffd83dbSDimitry Andric case X86::VPERMILPSZrm: 18245ffd83dbSDimitry Andric case X86::VPERMILPSZrmk: 1825*0fca6ea1SDimitry Andric case X86::VPERMILPSZrmkz: { 1826*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 1827*0fca6ea1SDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) { 1828*0fca6ea1SDimitry Andric unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 1829*0fca6ea1SDimitry Andric SmallVector<int, 16> Mask; 1830*0fca6ea1SDimitry Andric DecodeVPERMILPMask(C, 32, Width, Mask); 1831*0fca6ea1SDimitry Andric if (!Mask.empty()) 1832*0fca6ea1SDimitry Andric OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); 1833*0fca6ea1SDimitry Andric } 1834*0fca6ea1SDimitry Andric break; 1835*0fca6ea1SDimitry Andric } 18365ffd83dbSDimitry Andric case X86::VPERMILPDrm: 18375ffd83dbSDimitry Andric case X86::VPERMILPDYrm: 18385ffd83dbSDimitry Andric case X86::VPERMILPDZ128rm: 18395ffd83dbSDimitry Andric case X86::VPERMILPDZ128rmk: 18405ffd83dbSDimitry Andric case X86::VPERMILPDZ128rmkz: 18415ffd83dbSDimitry Andric case X86::VPERMILPDZ256rm: 18425ffd83dbSDimitry Andric case X86::VPERMILPDZ256rmk: 18435ffd83dbSDimitry Andric case X86::VPERMILPDZ256rmkz: 18445ffd83dbSDimitry Andric case X86::VPERMILPDZrm: 18455ffd83dbSDimitry Andric case X86::VPERMILPDZrmk: 18465ffd83dbSDimitry Andric case X86::VPERMILPDZrmkz: { 1847*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 18487a6dacacSDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) { 1849*0fca6ea1SDimitry Andric unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 18505ffd83dbSDimitry Andric SmallVector<int, 16> Mask; 1851*0fca6ea1SDimitry Andric DecodeVPERMILPMask(C, 64, Width, Mask); 18525ffd83dbSDimitry Andric if (!Mask.empty()) 18535ffd83dbSDimitry Andric OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask)); 18545ffd83dbSDimitry Andric } 18555ffd83dbSDimitry Andric break; 18565ffd83dbSDimitry Andric } 18575ffd83dbSDimitry Andric 18585ffd83dbSDimitry Andric case X86::VPERMIL2PDrm: 18595ffd83dbSDimitry Andric case X86::VPERMIL2PSrm: 18605ffd83dbSDimitry Andric case X86::VPERMIL2PDYrm: 18615ffd83dbSDimitry Andric case X86::VPERMIL2PSYrm: { 18625ffd83dbSDimitry Andric assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) && 18635ffd83dbSDimitry Andric "Unexpected number of operands!"); 18645ffd83dbSDimitry Andric 18655ffd83dbSDimitry Andric const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1); 18665ffd83dbSDimitry Andric if (!CtrlOp.isImm()) 18675ffd83dbSDimitry Andric break; 18685ffd83dbSDimitry Andric 18695ffd83dbSDimitry Andric unsigned ElSize; 18705ffd83dbSDimitry Andric switch (MI->getOpcode()) { 18715ffd83dbSDimitry Andric default: llvm_unreachable("Invalid opcode"); 18725ffd83dbSDimitry Andric case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break; 18735ffd83dbSDimitry Andric case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break; 18745ffd83dbSDimitry Andric } 18755ffd83dbSDimitry Andric 18767a6dacacSDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, 3)) { 1877*0fca6ea1SDimitry Andric unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 18785ffd83dbSDimitry Andric SmallVector<int, 16> Mask; 18795ffd83dbSDimitry Andric DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask); 18805ffd83dbSDimitry Andric if (!Mask.empty()) 18815ffd83dbSDimitry Andric OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask)); 18825ffd83dbSDimitry Andric } 18835ffd83dbSDimitry Andric break; 18845ffd83dbSDimitry Andric } 18855ffd83dbSDimitry Andric 18865ffd83dbSDimitry Andric case X86::VPPERMrrm: { 18877a6dacacSDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, 3)) { 1888*0fca6ea1SDimitry Andric unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 18895ffd83dbSDimitry Andric SmallVector<int, 16> Mask; 18905ffd83dbSDimitry Andric DecodeVPPERMMask(C, Width, Mask); 18915ffd83dbSDimitry Andric if (!Mask.empty()) 18925ffd83dbSDimitry Andric OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask)); 18935ffd83dbSDimitry Andric } 18945ffd83dbSDimitry Andric break; 18955ffd83dbSDimitry Andric } 18965ffd83dbSDimitry Andric 18975ffd83dbSDimitry Andric case X86::MMX_MOVQ64rm: { 18987a6dacacSDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, 1)) { 18995ffd83dbSDimitry Andric std::string Comment; 19005ffd83dbSDimitry Andric raw_string_ostream CS(Comment); 19015ffd83dbSDimitry Andric const MachineOperand &DstOp = MI->getOperand(0); 19025ffd83dbSDimitry Andric CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = "; 19035ffd83dbSDimitry Andric if (auto *CF = dyn_cast<ConstantFP>(C)) { 1904fe6060f1SDimitry Andric CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false); 19055ffd83dbSDimitry Andric OutStreamer.AddComment(CS.str()); 19065ffd83dbSDimitry Andric } 19075ffd83dbSDimitry Andric } 19085ffd83dbSDimitry Andric break; 19095ffd83dbSDimitry Andric } 19105ffd83dbSDimitry Andric 1911*0fca6ea1SDimitry Andric #define INSTR_CASE(Prefix, Instr, Suffix, Postfix) \ 1912*0fca6ea1SDimitry Andric case X86::Prefix##Instr##Suffix##rm##Postfix: 1913*0fca6ea1SDimitry Andric 1914*0fca6ea1SDimitry Andric #define CASE_ARITH_RM(Instr) \ 1915*0fca6ea1SDimitry Andric INSTR_CASE(, Instr, , ) /* SSE */ \ 1916*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, , ) /* AVX-128 */ \ 1917*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Y, ) /* AVX-256 */ \ 1918*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z128, ) \ 1919*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z128, k) \ 1920*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z128, kz) \ 1921*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z256, ) \ 1922*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z256, k) \ 1923*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z256, kz) \ 1924*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z, ) \ 1925*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z, k) \ 1926*0fca6ea1SDimitry Andric INSTR_CASE(V, Instr, Z, kz) 1927*0fca6ea1SDimitry Andric 1928*0fca6ea1SDimitry Andric // TODO: Add additional instructions when useful. 1929*0fca6ea1SDimitry Andric CASE_ARITH_RM(PMADDUBSW) { 1930*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 1931*0fca6ea1SDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) { 1932*0fca6ea1SDimitry Andric if (C->getType()->getScalarSizeInBits() == 8) { 1933*0fca6ea1SDimitry Andric std::string Comment; 1934*0fca6ea1SDimitry Andric raw_string_ostream CS(Comment); 1935*0fca6ea1SDimitry Andric unsigned VectorWidth = 1936*0fca6ea1SDimitry Andric X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 1937*0fca6ea1SDimitry Andric CS << "["; 1938*0fca6ea1SDimitry Andric printConstant(C, VectorWidth, CS); 1939*0fca6ea1SDimitry Andric CS << "]"; 1940*0fca6ea1SDimitry Andric OutStreamer.AddComment(CS.str()); 1941*0fca6ea1SDimitry Andric } 1942*0fca6ea1SDimitry Andric } 1943*0fca6ea1SDimitry Andric break; 1944*0fca6ea1SDimitry Andric } 1945*0fca6ea1SDimitry Andric 1946*0fca6ea1SDimitry Andric CASE_ARITH_RM(PMADDWD) 1947*0fca6ea1SDimitry Andric CASE_ARITH_RM(PMULLW) 1948*0fca6ea1SDimitry Andric CASE_ARITH_RM(PMULHW) 1949*0fca6ea1SDimitry Andric CASE_ARITH_RM(PMULHUW) 1950*0fca6ea1SDimitry Andric CASE_ARITH_RM(PMULHRSW) { 1951*0fca6ea1SDimitry Andric unsigned SrcIdx = getSrcIdx(MI, 1); 1952*0fca6ea1SDimitry Andric if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) { 1953*0fca6ea1SDimitry Andric if (C->getType()->getScalarSizeInBits() == 16) { 1954*0fca6ea1SDimitry Andric std::string Comment; 1955*0fca6ea1SDimitry Andric raw_string_ostream CS(Comment); 1956*0fca6ea1SDimitry Andric unsigned VectorWidth = 1957*0fca6ea1SDimitry Andric X86::getVectorRegisterWidth(MI->getDesc().operands()[0]); 1958*0fca6ea1SDimitry Andric CS << "["; 1959*0fca6ea1SDimitry Andric printConstant(C, VectorWidth, CS); 1960*0fca6ea1SDimitry Andric CS << "]"; 1961*0fca6ea1SDimitry Andric OutStreamer.AddComment(CS.str()); 1962*0fca6ea1SDimitry Andric } 1963*0fca6ea1SDimitry Andric } 1964*0fca6ea1SDimitry Andric break; 1965*0fca6ea1SDimitry Andric } 1966*0fca6ea1SDimitry Andric 1967*0fca6ea1SDimitry Andric #define MASK_AVX512_CASE(Instr) \ 1968*0fca6ea1SDimitry Andric case Instr: \ 1969*0fca6ea1SDimitry Andric case Instr##k: \ 1970*0fca6ea1SDimitry Andric case Instr##kz: 1971*0fca6ea1SDimitry Andric 19727a6dacacSDimitry Andric case X86::MOVSDrm: 19737a6dacacSDimitry Andric case X86::VMOVSDrm: 1974*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VMOVSDZrm) 19757a6dacacSDimitry Andric case X86::MOVSDrm_alt: 19767a6dacacSDimitry Andric case X86::VMOVSDrm_alt: 19777a6dacacSDimitry Andric case X86::VMOVSDZrm_alt: 19787a6dacacSDimitry Andric case X86::MOVQI2PQIrm: 19797a6dacacSDimitry Andric case X86::VMOVQI2PQIrm: 19807a6dacacSDimitry Andric case X86::VMOVQI2PQIZrm: 19817a6dacacSDimitry Andric printZeroUpperMove(MI, OutStreamer, 64, 128, "mem[0],zero"); 19827a6dacacSDimitry Andric break; 19837a6dacacSDimitry Andric 1984*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VMOVSHZrm) 1985*0fca6ea1SDimitry Andric case X86::VMOVSHZrm_alt: 1986*0fca6ea1SDimitry Andric printZeroUpperMove(MI, OutStreamer, 16, 128, 1987*0fca6ea1SDimitry Andric "mem[0],zero,zero,zero,zero,zero,zero,zero"); 1988*0fca6ea1SDimitry Andric break; 1989*0fca6ea1SDimitry Andric 19907a6dacacSDimitry Andric case X86::MOVSSrm: 19917a6dacacSDimitry Andric case X86::VMOVSSrm: 1992*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VMOVSSZrm) 19937a6dacacSDimitry Andric case X86::MOVSSrm_alt: 19947a6dacacSDimitry Andric case X86::VMOVSSrm_alt: 19957a6dacacSDimitry Andric case X86::VMOVSSZrm_alt: 19967a6dacacSDimitry Andric case X86::MOVDI2PDIrm: 19977a6dacacSDimitry Andric case X86::VMOVDI2PDIrm: 19987a6dacacSDimitry Andric case X86::VMOVDI2PDIZrm: 19997a6dacacSDimitry Andric printZeroUpperMove(MI, OutStreamer, 32, 128, "mem[0],zero,zero,zero"); 20007a6dacacSDimitry Andric break; 20017a6dacacSDimitry Andric 20025ffd83dbSDimitry Andric #define MOV_CASE(Prefix, Suffix) \ 20035ffd83dbSDimitry Andric case X86::Prefix##MOVAPD##Suffix##rm: \ 20045ffd83dbSDimitry Andric case X86::Prefix##MOVAPS##Suffix##rm: \ 20055ffd83dbSDimitry Andric case X86::Prefix##MOVUPD##Suffix##rm: \ 20065ffd83dbSDimitry Andric case X86::Prefix##MOVUPS##Suffix##rm: \ 20075ffd83dbSDimitry Andric case X86::Prefix##MOVDQA##Suffix##rm: \ 20085ffd83dbSDimitry Andric case X86::Prefix##MOVDQU##Suffix##rm: 20095ffd83dbSDimitry Andric 2010*0fca6ea1SDimitry Andric #define MOV_AVX512_CASE(Suffix, Postfix) \ 2011*0fca6ea1SDimitry Andric case X86::VMOVDQA64##Suffix##rm##Postfix: \ 2012*0fca6ea1SDimitry Andric case X86::VMOVDQA32##Suffix##rm##Postfix: \ 2013*0fca6ea1SDimitry Andric case X86::VMOVDQU64##Suffix##rm##Postfix: \ 2014*0fca6ea1SDimitry Andric case X86::VMOVDQU32##Suffix##rm##Postfix: \ 2015*0fca6ea1SDimitry Andric case X86::VMOVDQU16##Suffix##rm##Postfix: \ 2016*0fca6ea1SDimitry Andric case X86::VMOVDQU8##Suffix##rm##Postfix: \ 2017*0fca6ea1SDimitry Andric case X86::VMOVAPS##Suffix##rm##Postfix: \ 2018*0fca6ea1SDimitry Andric case X86::VMOVAPD##Suffix##rm##Postfix: \ 2019*0fca6ea1SDimitry Andric case X86::VMOVUPS##Suffix##rm##Postfix: \ 2020*0fca6ea1SDimitry Andric case X86::VMOVUPD##Suffix##rm##Postfix: 20215ffd83dbSDimitry Andric 20225f757f3fSDimitry Andric #define CASE_128_MOV_RM() \ 20235f757f3fSDimitry Andric MOV_CASE(, ) /* SSE */ \ 20245f757f3fSDimitry Andric MOV_CASE(V, ) /* AVX-128 */ \ 2025*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z128, ) \ 2026*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z128, k) \ 2027*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z128, kz) 20285f757f3fSDimitry Andric 20295f757f3fSDimitry Andric #define CASE_256_MOV_RM() \ 20305f757f3fSDimitry Andric MOV_CASE(V, Y) /* AVX-256 */ \ 2031*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z256, ) \ 2032*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z256, k) \ 2033*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z256, kz) \ 20345f757f3fSDimitry Andric 20355f757f3fSDimitry Andric #define CASE_512_MOV_RM() \ 2036*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z, ) \ 2037*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z, k) \ 2038*0fca6ea1SDimitry Andric MOV_AVX512_CASE(Z, kz) \ 20395f757f3fSDimitry Andric 20405ffd83dbSDimitry Andric // For loads from a constant pool to a vector register, print the constant 20415ffd83dbSDimitry Andric // loaded. 20427a6dacacSDimitry Andric CASE_128_MOV_RM() 20437a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 1, 128); 20447a6dacacSDimitry Andric break; 20457a6dacacSDimitry Andric CASE_256_MOV_RM() 20467a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 1, 256); 20477a6dacacSDimitry Andric break; 20487a6dacacSDimitry Andric CASE_512_MOV_RM() 20497a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 1, 512); 20507a6dacacSDimitry Andric break; 20515f757f3fSDimitry Andric case X86::VBROADCASTF128rm: 20525f757f3fSDimitry Andric case X86::VBROADCASTI128rm: 2053*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTF32X4Z256rm) 2054*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTF64X2Z128rm) 2055*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTI32X4Z256rm) 2056*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTI64X2Z128rm) 20577a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 2, 128); 20587a6dacacSDimitry Andric break; 2059*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTF32X4rm) 2060*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTF64X2rm) 2061*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTI32X4rm) 2062*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTI64X2rm) 20637a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 4, 128); 20647a6dacacSDimitry Andric break; 2065*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTF32X8rm) 2066*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTF64X4rm) 2067*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTI32X8rm) 2068*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTI64X4rm) 20697a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 2, 256); 20705ffd83dbSDimitry Andric break; 20715ffd83dbSDimitry Andric 20727a6dacacSDimitry Andric // For broadcast loads from a constant pool to a vector register, repeatedly 20737a6dacacSDimitry Andric // print the constant loaded. 20745ffd83dbSDimitry Andric case X86::MOVDDUPrm: 20755ffd83dbSDimitry Andric case X86::VMOVDDUPrm: 2076*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VMOVDDUPZ128rm) 20777a6dacacSDimitry Andric case X86::VPBROADCASTQrm: 2078*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTQZ128rm) 20797a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 2, 64); 20807a6dacacSDimitry Andric break; 20815ffd83dbSDimitry Andric case X86::VBROADCASTSDYrm: 2082*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTSDZ256rm) 20835ffd83dbSDimitry Andric case X86::VPBROADCASTQYrm: 2084*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTQZ256rm) 20857a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 4, 64); 20867a6dacacSDimitry Andric break; 2087*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTSDZrm) 2088*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTQZrm) 20897a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 8, 64); 20907a6dacacSDimitry Andric break; 20917a6dacacSDimitry Andric case X86::VBROADCASTSSrm: 2092*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTSSZ128rm) 20937a6dacacSDimitry Andric case X86::VPBROADCASTDrm: 2094*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTDZ128rm) 20957a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 4, 32); 20967a6dacacSDimitry Andric break; 20977a6dacacSDimitry Andric case X86::VBROADCASTSSYrm: 2098*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTSSZ256rm) 20997a6dacacSDimitry Andric case X86::VPBROADCASTDYrm: 2100*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTDZ256rm) 21017a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 8, 32); 21027a6dacacSDimitry Andric break; 2103*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VBROADCASTSSZrm) 2104*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTDZrm) 21057a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 16, 32); 21067a6dacacSDimitry Andric break; 21075ffd83dbSDimitry Andric case X86::VPBROADCASTWrm: 2108*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTWZ128rm) 21097a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 8, 16); 21107a6dacacSDimitry Andric break; 21117a6dacacSDimitry Andric case X86::VPBROADCASTWYrm: 2112*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTWZ256rm) 21137a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 16, 16); 21147a6dacacSDimitry Andric break; 2115*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTWZrm) 21167a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 32, 16); 21177a6dacacSDimitry Andric break; 21187a6dacacSDimitry Andric case X86::VPBROADCASTBrm: 2119*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTBZ128rm) 21207a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 16, 8); 21217a6dacacSDimitry Andric break; 21227a6dacacSDimitry Andric case X86::VPBROADCASTBYrm: 2123*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTBZ256rm) 21247a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 32, 8); 21257a6dacacSDimitry Andric break; 2126*0fca6ea1SDimitry Andric MASK_AVX512_CASE(X86::VPBROADCASTBZrm) 21277a6dacacSDimitry Andric printBroadcast(MI, OutStreamer, 64, 8); 21287a6dacacSDimitry Andric break; 2129*0fca6ea1SDimitry Andric 2130*0fca6ea1SDimitry Andric #define MOVX_CASE(Prefix, Ext, Type, Suffix, Postfix) \ 2131*0fca6ea1SDimitry Andric case X86::Prefix##PMOV##Ext##Type##Suffix##rm##Postfix: 2132*0fca6ea1SDimitry Andric 2133*0fca6ea1SDimitry Andric #define CASE_MOVX_RM(Ext, Type) \ 2134*0fca6ea1SDimitry Andric MOVX_CASE(, Ext, Type, , ) \ 2135*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, , ) \ 2136*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Y, ) \ 2137*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z128, ) \ 2138*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z128, k ) \ 2139*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z128, kz ) \ 2140*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z256, ) \ 2141*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z256, k ) \ 2142*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z256, kz ) \ 2143*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z, ) \ 2144*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z, k ) \ 2145*0fca6ea1SDimitry Andric MOVX_CASE(V, Ext, Type, Z, kz ) 2146*0fca6ea1SDimitry Andric 2147*0fca6ea1SDimitry Andric CASE_MOVX_RM(SX, BD) 2148*0fca6ea1SDimitry Andric printSignExtend(MI, OutStreamer, 8, 32); 2149*0fca6ea1SDimitry Andric break; 2150*0fca6ea1SDimitry Andric CASE_MOVX_RM(SX, BQ) 2151*0fca6ea1SDimitry Andric printSignExtend(MI, OutStreamer, 8, 64); 2152*0fca6ea1SDimitry Andric break; 2153*0fca6ea1SDimitry Andric CASE_MOVX_RM(SX, BW) 2154*0fca6ea1SDimitry Andric printSignExtend(MI, OutStreamer, 8, 16); 2155*0fca6ea1SDimitry Andric break; 2156*0fca6ea1SDimitry Andric CASE_MOVX_RM(SX, DQ) 2157*0fca6ea1SDimitry Andric printSignExtend(MI, OutStreamer, 32, 64); 2158*0fca6ea1SDimitry Andric break; 2159*0fca6ea1SDimitry Andric CASE_MOVX_RM(SX, WD) 2160*0fca6ea1SDimitry Andric printSignExtend(MI, OutStreamer, 16, 32); 2161*0fca6ea1SDimitry Andric break; 2162*0fca6ea1SDimitry Andric CASE_MOVX_RM(SX, WQ) 2163*0fca6ea1SDimitry Andric printSignExtend(MI, OutStreamer, 16, 64); 2164*0fca6ea1SDimitry Andric break; 2165*0fca6ea1SDimitry Andric 2166*0fca6ea1SDimitry Andric CASE_MOVX_RM(ZX, BD) 2167*0fca6ea1SDimitry Andric printZeroExtend(MI, OutStreamer, 8, 32); 2168*0fca6ea1SDimitry Andric break; 2169*0fca6ea1SDimitry Andric CASE_MOVX_RM(ZX, BQ) 2170*0fca6ea1SDimitry Andric printZeroExtend(MI, OutStreamer, 8, 64); 2171*0fca6ea1SDimitry Andric break; 2172*0fca6ea1SDimitry Andric CASE_MOVX_RM(ZX, BW) 2173*0fca6ea1SDimitry Andric printZeroExtend(MI, OutStreamer, 8, 16); 2174*0fca6ea1SDimitry Andric break; 2175*0fca6ea1SDimitry Andric CASE_MOVX_RM(ZX, DQ) 2176*0fca6ea1SDimitry Andric printZeroExtend(MI, OutStreamer, 32, 64); 2177*0fca6ea1SDimitry Andric break; 2178*0fca6ea1SDimitry Andric CASE_MOVX_RM(ZX, WD) 2179*0fca6ea1SDimitry Andric printZeroExtend(MI, OutStreamer, 16, 32); 2180*0fca6ea1SDimitry Andric break; 2181*0fca6ea1SDimitry Andric CASE_MOVX_RM(ZX, WQ) 2182*0fca6ea1SDimitry Andric printZeroExtend(MI, OutStreamer, 16, 64); 2183*0fca6ea1SDimitry Andric break; 21845ffd83dbSDimitry Andric } 21855ffd83dbSDimitry Andric } 21865ffd83dbSDimitry Andric 21875ffd83dbSDimitry Andric void X86AsmPrinter::emitInstruction(const MachineInstr *MI) { 2188753f127fSDimitry Andric // FIXME: Enable feature predicate checks once all the test pass. 2189753f127fSDimitry Andric // X86_MC::verifyInstructionPredicates(MI->getOpcode(), 2190753f127fSDimitry Andric // Subtarget->getFeatureBits()); 2191753f127fSDimitry Andric 21920b57cec5SDimitry Andric X86MCInstLower MCInstLowering(*MF, *this); 21930b57cec5SDimitry Andric const X86RegisterInfo *RI = 21940b57cec5SDimitry Andric MF->getSubtarget<X86Subtarget>().getRegisterInfo(); 21950b57cec5SDimitry Andric 21960eae32dcSDimitry Andric if (MI->getOpcode() == X86::OR64rm) { 21970eae32dcSDimitry Andric for (auto &Opd : MI->operands()) { 21980eae32dcSDimitry Andric if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) == 21990eae32dcSDimitry Andric "swift_async_extendedFramePointerFlags") { 22000eae32dcSDimitry Andric ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true; 22010eae32dcSDimitry Andric } 22020eae32dcSDimitry Andric } 22030eae32dcSDimitry Andric } 22040eae32dcSDimitry Andric 22057a6dacacSDimitry Andric // Add comments for values loaded from constant pool. 22067a6dacacSDimitry Andric if (OutStreamer->isVerboseAsm()) 22077a6dacacSDimitry Andric addConstantComments(MI, *OutStreamer); 22087a6dacacSDimitry Andric 22091db9f3b2SDimitry Andric // Add a comment about EVEX compression 22100b57cec5SDimitry Andric if (TM.Options.MCOptions.ShowMCEncoding) { 22111db9f3b2SDimitry Andric if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY) 22121db9f3b2SDimitry Andric OutStreamer->AddComment("EVEX TO LEGACY Compression ", false); 22131db9f3b2SDimitry Andric else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX) 22140b57cec5SDimitry Andric OutStreamer->AddComment("EVEX TO VEX Compression ", false); 22157a6dacacSDimitry Andric else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX) 22167a6dacacSDimitry Andric OutStreamer->AddComment("EVEX TO EVEX Compression ", false); 22170b57cec5SDimitry Andric } 22180b57cec5SDimitry Andric 22190b57cec5SDimitry Andric switch (MI->getOpcode()) { 22200b57cec5SDimitry Andric case TargetOpcode::DBG_VALUE: 22210b57cec5SDimitry Andric llvm_unreachable("Should be handled target independently"); 22220b57cec5SDimitry Andric 22230b57cec5SDimitry Andric case X86::EH_RETURN: 22240b57cec5SDimitry Andric case X86::EH_RETURN64: { 22250b57cec5SDimitry Andric // Lower these as normal, but add some comments. 22268bcb0991SDimitry Andric Register Reg = MI->getOperand(0).getReg(); 22270b57cec5SDimitry Andric OutStreamer->AddComment(StringRef("eh_return, addr: %") + 22280b57cec5SDimitry Andric X86ATTInstPrinter::getRegisterName(Reg)); 22290b57cec5SDimitry Andric break; 22300b57cec5SDimitry Andric } 22310b57cec5SDimitry Andric case X86::CLEANUPRET: { 22320b57cec5SDimitry Andric // Lower these as normal, but add some comments. 22330b57cec5SDimitry Andric OutStreamer->AddComment("CLEANUPRET"); 22340b57cec5SDimitry Andric break; 22350b57cec5SDimitry Andric } 22360b57cec5SDimitry Andric 22370b57cec5SDimitry Andric case X86::CATCHRET: { 22380b57cec5SDimitry Andric // Lower these as normal, but add some comments. 22390b57cec5SDimitry Andric OutStreamer->AddComment("CATCHRET"); 22400b57cec5SDimitry Andric break; 22410b57cec5SDimitry Andric } 22420b57cec5SDimitry Andric 224313138422SDimitry Andric case X86::ENDBR32: 224413138422SDimitry Andric case X86::ENDBR64: { 224513138422SDimitry Andric // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for 224613138422SDimitry Andric // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be 224713138422SDimitry Andric // non-empty. If MI is the initial ENDBR, place the 224813138422SDimitry Andric // __patchable_function_entries label after ENDBR. 224913138422SDimitry Andric if (CurrentPatchableFunctionEntrySym && 225013138422SDimitry Andric CurrentPatchableFunctionEntrySym == CurrentFnBegin && 225113138422SDimitry Andric MI == &MF->front().front()) { 225213138422SDimitry Andric MCInst Inst; 225313138422SDimitry Andric MCInstLowering.Lower(MI, Inst); 225413138422SDimitry Andric EmitAndCountInstruction(Inst); 225513138422SDimitry Andric CurrentPatchableFunctionEntrySym = createTempSymbol("patch"); 22565ffd83dbSDimitry Andric OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym); 225713138422SDimitry Andric return; 225813138422SDimitry Andric } 225913138422SDimitry Andric break; 226013138422SDimitry Andric } 226113138422SDimitry Andric 2262bdd1243dSDimitry Andric case X86::TAILJMPd64: 2263bdd1243dSDimitry Andric if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11)) 2264bdd1243dSDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX)); 2265bdd1243dSDimitry Andric [[fallthrough]]; 22660b57cec5SDimitry Andric case X86::TAILJMPr: 22670b57cec5SDimitry Andric case X86::TAILJMPm: 22680b57cec5SDimitry Andric case X86::TAILJMPd: 22690b57cec5SDimitry Andric case X86::TAILJMPd_CC: 22700b57cec5SDimitry Andric case X86::TAILJMPr64: 22710b57cec5SDimitry Andric case X86::TAILJMPm64: 22720b57cec5SDimitry Andric case X86::TAILJMPd64_CC: 22730b57cec5SDimitry Andric case X86::TAILJMPr64_REX: 22740b57cec5SDimitry Andric case X86::TAILJMPm64_REX: 22750b57cec5SDimitry Andric // Lower these as normal, but add some comments. 22760b57cec5SDimitry Andric OutStreamer->AddComment("TAILCALL"); 22770b57cec5SDimitry Andric break; 22780b57cec5SDimitry Andric 22790b57cec5SDimitry Andric case X86::TLS_addr32: 22800b57cec5SDimitry Andric case X86::TLS_addr64: 2281e8d8bef9SDimitry Andric case X86::TLS_addrX32: 22820b57cec5SDimitry Andric case X86::TLS_base_addr32: 22830b57cec5SDimitry Andric case X86::TLS_base_addr64: 2284e8d8bef9SDimitry Andric case X86::TLS_base_addrX32: 2285*0fca6ea1SDimitry Andric case X86::TLS_desc32: 2286*0fca6ea1SDimitry Andric case X86::TLS_desc64: 22870b57cec5SDimitry Andric return LowerTlsAddr(MCInstLowering, *MI); 22880b57cec5SDimitry Andric 22890b57cec5SDimitry Andric case X86::MOVPC32r: { 22900b57cec5SDimitry Andric // This is a pseudo op for a two instruction sequence with a label, which 22910b57cec5SDimitry Andric // looks like: 22920b57cec5SDimitry Andric // call "L1$pb" 22930b57cec5SDimitry Andric // "L1$pb": 22940b57cec5SDimitry Andric // popl %esi 22950b57cec5SDimitry Andric 22960b57cec5SDimitry Andric // Emit the call. 22970b57cec5SDimitry Andric MCSymbol *PICBase = MF->getPICBaseSymbol(); 22980b57cec5SDimitry Andric // FIXME: We would like an efficient form for this, so we don't have to do a 22990b57cec5SDimitry Andric // lot of extra uniquing. 23000b57cec5SDimitry Andric EmitAndCountInstruction( 23010b57cec5SDimitry Andric MCInstBuilder(X86::CALLpcrel32) 23020b57cec5SDimitry Andric .addExpr(MCSymbolRefExpr::create(PICBase, OutContext))); 23030b57cec5SDimitry Andric 23040b57cec5SDimitry Andric const X86FrameLowering *FrameLowering = 23050b57cec5SDimitry Andric MF->getSubtarget<X86Subtarget>().getFrameLowering(); 23060b57cec5SDimitry Andric bool hasFP = FrameLowering->hasFP(*MF); 23070b57cec5SDimitry Andric 23080b57cec5SDimitry Andric // TODO: This is needed only if we require precise CFA. 23090b57cec5SDimitry Andric bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() && 23100b57cec5SDimitry Andric !OutStreamer->getDwarfFrameInfos().back().End; 23110b57cec5SDimitry Andric 23120b57cec5SDimitry Andric int stackGrowth = -RI->getSlotSize(); 23130b57cec5SDimitry Andric 23140b57cec5SDimitry Andric if (HasActiveDwarfFrame && !hasFP) { 23155ffd83dbSDimitry Andric OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth); 231606c3fb27SDimitry Andric MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true); 23170b57cec5SDimitry Andric } 23180b57cec5SDimitry Andric 23190b57cec5SDimitry Andric // Emit the label. 23205ffd83dbSDimitry Andric OutStreamer->emitLabel(PICBase); 23210b57cec5SDimitry Andric 23220b57cec5SDimitry Andric // popl $reg 23230b57cec5SDimitry Andric EmitAndCountInstruction( 23240b57cec5SDimitry Andric MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg())); 23250b57cec5SDimitry Andric 23260b57cec5SDimitry Andric if (HasActiveDwarfFrame && !hasFP) { 23275ffd83dbSDimitry Andric OutStreamer->emitCFIAdjustCfaOffset(stackGrowth); 23280b57cec5SDimitry Andric } 23290b57cec5SDimitry Andric return; 23300b57cec5SDimitry Andric } 23310b57cec5SDimitry Andric 23320b57cec5SDimitry Andric case X86::ADD32ri: { 23330b57cec5SDimitry Andric // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri. 23340b57cec5SDimitry Andric if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS) 23350b57cec5SDimitry Andric break; 23360b57cec5SDimitry Andric 23370b57cec5SDimitry Andric // Okay, we have something like: 23380b57cec5SDimitry Andric // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL) 23390b57cec5SDimitry Andric 23400b57cec5SDimitry Andric // For this, we want to print something like: 23410b57cec5SDimitry Andric // MYGLOBAL + (. - PICBASE) 23420b57cec5SDimitry Andric // However, we can't generate a ".", so just emit a new label here and refer 23430b57cec5SDimitry Andric // to it. 23440b57cec5SDimitry Andric MCSymbol *DotSym = OutContext.createTempSymbol(); 23455ffd83dbSDimitry Andric OutStreamer->emitLabel(DotSym); 23460b57cec5SDimitry Andric 23470b57cec5SDimitry Andric // Now that we have emitted the label, lower the complex operand expression. 23480b57cec5SDimitry Andric MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2)); 23490b57cec5SDimitry Andric 23500b57cec5SDimitry Andric const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext); 23510b57cec5SDimitry Andric const MCExpr *PICBase = 23520b57cec5SDimitry Andric MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext); 23530b57cec5SDimitry Andric DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext); 23540b57cec5SDimitry Andric 23550b57cec5SDimitry Andric DotExpr = MCBinaryExpr::createAdd( 23560b57cec5SDimitry Andric MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext); 23570b57cec5SDimitry Andric 23580b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri) 23590b57cec5SDimitry Andric .addReg(MI->getOperand(0).getReg()) 23600b57cec5SDimitry Andric .addReg(MI->getOperand(1).getReg()) 23610b57cec5SDimitry Andric .addExpr(DotExpr)); 23620b57cec5SDimitry Andric return; 23630b57cec5SDimitry Andric } 23640b57cec5SDimitry Andric case TargetOpcode::STATEPOINT: 23650b57cec5SDimitry Andric return LowerSTATEPOINT(*MI, MCInstLowering); 23660b57cec5SDimitry Andric 23670b57cec5SDimitry Andric case TargetOpcode::FAULTING_OP: 23680b57cec5SDimitry Andric return LowerFAULTING_OP(*MI, MCInstLowering); 23690b57cec5SDimitry Andric 23700b57cec5SDimitry Andric case TargetOpcode::FENTRY_CALL: 23710b57cec5SDimitry Andric return LowerFENTRY_CALL(*MI, MCInstLowering); 23720b57cec5SDimitry Andric 23730b57cec5SDimitry Andric case TargetOpcode::PATCHABLE_OP: 23740b57cec5SDimitry Andric return LowerPATCHABLE_OP(*MI, MCInstLowering); 23750b57cec5SDimitry Andric 23760b57cec5SDimitry Andric case TargetOpcode::STACKMAP: 23770b57cec5SDimitry Andric return LowerSTACKMAP(*MI); 23780b57cec5SDimitry Andric 23790b57cec5SDimitry Andric case TargetOpcode::PATCHPOINT: 23800b57cec5SDimitry Andric return LowerPATCHPOINT(*MI, MCInstLowering); 23810b57cec5SDimitry Andric 23820b57cec5SDimitry Andric case TargetOpcode::PATCHABLE_FUNCTION_ENTER: 23830b57cec5SDimitry Andric return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering); 23840b57cec5SDimitry Andric 23850b57cec5SDimitry Andric case TargetOpcode::PATCHABLE_RET: 23860b57cec5SDimitry Andric return LowerPATCHABLE_RET(*MI, MCInstLowering); 23870b57cec5SDimitry Andric 23880b57cec5SDimitry Andric case TargetOpcode::PATCHABLE_TAIL_CALL: 23890b57cec5SDimitry Andric return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering); 23900b57cec5SDimitry Andric 23910b57cec5SDimitry Andric case TargetOpcode::PATCHABLE_EVENT_CALL: 23920b57cec5SDimitry Andric return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering); 23930b57cec5SDimitry Andric 23940b57cec5SDimitry Andric case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL: 23950b57cec5SDimitry Andric return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering); 23960b57cec5SDimitry Andric 23970b57cec5SDimitry Andric case X86::MORESTACK_RET: 23980b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); 23990b57cec5SDimitry Andric return; 24000b57cec5SDimitry Andric 2401bdd1243dSDimitry Andric case X86::KCFI_CHECK: 2402bdd1243dSDimitry Andric return LowerKCFI_CHECK(*MI); 2403bdd1243dSDimitry Andric 2404349cc55cSDimitry Andric case X86::ASAN_CHECK_MEMACCESS: 2405349cc55cSDimitry Andric return LowerASAN_CHECK_MEMACCESS(*MI); 2406349cc55cSDimitry Andric 24070b57cec5SDimitry Andric case X86::MORESTACK_RET_RESTORE_R10: 24080b57cec5SDimitry Andric // Return, then restore R10. 24090b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget))); 24100b57cec5SDimitry Andric EmitAndCountInstruction( 24110b57cec5SDimitry Andric MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX)); 24120b57cec5SDimitry Andric return; 24130b57cec5SDimitry Andric 24140b57cec5SDimitry Andric case X86::SEH_PushReg: 24150b57cec5SDimitry Andric case X86::SEH_SaveReg: 24160b57cec5SDimitry Andric case X86::SEH_SaveXMM: 24170b57cec5SDimitry Andric case X86::SEH_StackAlloc: 24180b57cec5SDimitry Andric case X86::SEH_StackAlign: 24190b57cec5SDimitry Andric case X86::SEH_SetFrame: 24200b57cec5SDimitry Andric case X86::SEH_PushFrame: 24210b57cec5SDimitry Andric case X86::SEH_EndPrologue: 24220b57cec5SDimitry Andric EmitSEHInstruction(MI); 24230b57cec5SDimitry Andric return; 24240b57cec5SDimitry Andric 24250b57cec5SDimitry Andric case X86::SEH_Epilogue: { 24260b57cec5SDimitry Andric assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?"); 24270b57cec5SDimitry Andric MachineBasicBlock::const_iterator MBBI(MI); 24280b57cec5SDimitry Andric // Check if preceded by a call and emit nop if so. 24290b57cec5SDimitry Andric for (MBBI = PrevCrossBBInst(MBBI); 24300b57cec5SDimitry Andric MBBI != MachineBasicBlock::const_iterator(); 24310b57cec5SDimitry Andric MBBI = PrevCrossBBInst(MBBI)) { 243206c3fb27SDimitry Andric // Pseudo instructions that aren't a call are assumed to not emit any 243306c3fb27SDimitry Andric // code. If they do, we worst case generate unnecessary noops after a 243406c3fb27SDimitry Andric // call. 243506c3fb27SDimitry Andric if (MBBI->isCall() || !MBBI->isPseudo()) { 24360b57cec5SDimitry Andric if (MBBI->isCall()) 24370b57cec5SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::NOOP)); 24380b57cec5SDimitry Andric break; 24390b57cec5SDimitry Andric } 24400b57cec5SDimitry Andric } 24410b57cec5SDimitry Andric return; 24420b57cec5SDimitry Andric } 2443e8d8bef9SDimitry Andric case X86::UBSAN_UD1: 2444e8d8bef9SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm) 2445e8d8bef9SDimitry Andric .addReg(X86::EAX) 2446e8d8bef9SDimitry Andric .addReg(X86::EAX) 2447e8d8bef9SDimitry Andric .addImm(1) 2448e8d8bef9SDimitry Andric .addReg(X86::NoRegister) 2449e8d8bef9SDimitry Andric .addImm(MI->getOperand(0).getImm()) 2450e8d8bef9SDimitry Andric .addReg(X86::NoRegister)); 2451e8d8bef9SDimitry Andric return; 2452bdd1243dSDimitry Andric case X86::CALL64pcrel32: 2453bdd1243dSDimitry Andric if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11)) 2454bdd1243dSDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX)); 2455bdd1243dSDimitry Andric break; 2456*0fca6ea1SDimitry Andric case X86::JCC_1: 2457*0fca6ea1SDimitry Andric // Two instruction prefixes (2EH for branch not-taken and 3EH for branch 2458*0fca6ea1SDimitry Andric // taken) are used as branch hints. Here we add branch taken prefix for 2459*0fca6ea1SDimitry Andric // jump instruction with higher probability than threshold. 2460*0fca6ea1SDimitry Andric if (getSubtarget().hasBranchHint() && EnableBranchHint) { 2461*0fca6ea1SDimitry Andric const MachineBranchProbabilityInfo *MBPI = 2462*0fca6ea1SDimitry Andric &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI(); 2463*0fca6ea1SDimitry Andric MachineBasicBlock *DestBB = MI->getOperand(0).getMBB(); 2464*0fca6ea1SDimitry Andric BranchProbability EdgeProb = 2465*0fca6ea1SDimitry Andric MBPI->getEdgeProbability(MI->getParent(), DestBB); 2466*0fca6ea1SDimitry Andric BranchProbability Threshold(BranchHintProbabilityThreshold, 100); 2467*0fca6ea1SDimitry Andric if (EdgeProb > Threshold) 2468*0fca6ea1SDimitry Andric EmitAndCountInstruction(MCInstBuilder(X86::DS_PREFIX)); 2469*0fca6ea1SDimitry Andric } 2470*0fca6ea1SDimitry Andric break; 24710b57cec5SDimitry Andric } 24720b57cec5SDimitry Andric 24730b57cec5SDimitry Andric MCInst TmpInst; 24740b57cec5SDimitry Andric MCInstLowering.Lower(MI, TmpInst); 24750b57cec5SDimitry Andric 24760b57cec5SDimitry Andric // Stackmap shadows cannot include branch targets, so we can count the bytes 24770b57cec5SDimitry Andric // in a call towards the shadow, but must ensure that the no thread returns 24780b57cec5SDimitry Andric // in to the stackmap shadow. The only way to achieve this is if the call 24790b57cec5SDimitry Andric // is at the end of the shadow. 24800b57cec5SDimitry Andric if (MI->isCall()) { 24810b57cec5SDimitry Andric // Count then size of the call towards the shadow 24820b57cec5SDimitry Andric SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get()); 24830b57cec5SDimitry Andric // Then flush the shadow so that we fill with nops before the call, not 24840b57cec5SDimitry Andric // after it. 24850b57cec5SDimitry Andric SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo()); 24860b57cec5SDimitry Andric // Then emit the call 24875ffd83dbSDimitry Andric OutStreamer->emitInstruction(TmpInst, getSubtargetInfo()); 24880b57cec5SDimitry Andric return; 24890b57cec5SDimitry Andric } 24900b57cec5SDimitry Andric 24910b57cec5SDimitry Andric EmitAndCountInstruction(TmpInst); 24920b57cec5SDimitry Andric } 2493