Lines Matching +full:is +full:- +full:decoded +full:- +full:cs
1 //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
58 static cl::opt<bool> EnableBranchHint("enable-branch-hint",
62 "branch-hint-probability-threshold",
68 /// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
123 CodeEmitter->encodeInstruction(Inst, Code, Fixups, STI);
126 InShadow = false; // The shadow is big enough. Stop counting.
134 emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,
135 &MF->getSubtarget<X86Subtarget>());
140 OutStreamer->emitInstruction(Inst, getSubtargetInfo());
150 return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();
153 /// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
192 Sym = MO.getMBB()->getSymbol();
206 AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();
223 !MO.getGlobal()->hasInternalLinkage());
314 // If .set directive is supported, use it to reduce the number of
316 // local labels. This is only safe when the symbols are in the same
319 AsmPrinter.OutStreamer->emitAssignment(Label, Expr);
342 MI->print(errs());
406 OutMI.setOpcode(MI->getOpcode());
408 for (const MachineOperand &MO : MI->operands())
413 if (X86::optimizeInstFromVEX3ToVEX2(OutMI, MI->getDesc()) ||
452 // CALL64r, CALL64pcrel32 - These instructions used to have
481 // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
519 MI->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
520 if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead())
534 MCContext &Ctx = OutStreamer->getContext();
562 // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
564 // only using GOT when GOTPCRELX is enabled.
566 bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
567 Ctx.getTargetOptions()->X86RelaxRelocations;
663 // Determine the longest nop which can be efficiently decoded for the given
664 // target cpu. 15-bytes is the longest single NOP instruction, but some
667 if (Subtarget->is64Bit()) {
668 // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
670 if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))
672 else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))
674 else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))
678 } if (Subtarget->is32Bit())
744 SegmentReg = X86::CS;
748 unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);
777 /// Emit the optimal amount of multi-byte nops on X86.
783 NumBytes -= emitNop(OS, NumBytes, Subtarget);
790 assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
811 // symbol is to far away. (TODO: support non-relative addressing)
819 // address is to far away. (TODO: support non-relative addressing)
823 if (Subtarget->useIndirectThunkCalls())
838 OutStreamer->emitInstruction(CallInst, getSubtargetInfo());
843 auto &Ctx = OutStreamer->getContext();
845 OutStreamer->emitLabel(MILabel);
859 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
863 auto &Ctx = OutStreamer->getContext();
865 OutStreamer->emitLabel(FaultingLabel);
881 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
882 OutStreamer->emitInstruction(MI, getSubtargetInfo());
887 bool Is64Bits = Subtarget->is64Bit();
888 MCContext &Ctx = OutStreamer->getContext();
899 assert(std::next(MI.getIterator())->isCall() &&
902 // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()
903 // returns a 1-byte X86::NOOP, which means the offset is the same in
904 // bytes. This assumes that patchable-function-prefix is the same for all
909 .getFnAttribute("patchable-function-prefix")
920 // The check is immediately before the call. If the call target is in R10,
924 MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type)));
931 .addImm(-(PrefixNops + 4))
941 OutStreamer->emitLabel(Trap);
944 OutStreamer->emitLabel(Pass);
948 // FIXME: Make this work on non-ELF.
968 TM.getMCRegisterInfo()->getName(Reg.asMCReg()))
972 "OrShadowOffset is not supported with optimized callbacks");
987 MI.getParent()->end().getInstrIterator(),
993 if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) {
995 // If the next instruction is inline assembly, we skip lowering it for now,
1001 CodeEmitter->encodeInstruction(MCI, Code, Fixups, getSubtargetInfo());
1005 if (MinSize == 2 && Subtarget->is32Bit() &&
1006 Subtarget->isTargetWindowsMSVC() &&
1007 (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1008 // For compatibility reasons, when targetting MSVC, it is important to
1011 // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1012 OutStreamer->emitInstruction(
1028 auto &Ctx = OutStreamer->getContext();
1030 OutStreamer->emitLabel(MILabel);
1041 assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1047 auto &Ctx = OutStreamer->getContext();
1049 OutStreamer->emitLabel(MILabel);
1057 // Check for null target. If target is non-null (i.e. is non-zero or is
1077 // This is encoded with 12-13 bytes, depending on which register is used.
1087 if (Subtarget->useIndirectThunkCalls())
1098 emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);
1103 assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1120 // nopw (2-byte nop)
1125 // ---
1128 OutStreamer->AddComment("# XRay Custom Event Log");
1129 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1130 OutStreamer->emitLabel(CurSled);
1132 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1135 OutStreamer->emitBinaryData("\xeb\x0f");
1138 // %rdx -- so we only work with those.
1165 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1173 // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1184 // Restore caller-saved and used registers.
1185 for (unsigned I = sizeof UsedMask; I-- > 0;)
1191 OutStreamer->AddComment("xray custom event end.");
1195 // changed the absolute address to a PC-relative address.
1201 assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1218 // nopw (2-byte nop)
1223 // ---
1226 OutStreamer->AddComment("# XRay Typed Event Log");
1227 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1228 OutStreamer->emitLabel(CurSled);
1230 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1233 OutStreamer->emitBinaryData("\xeb\x14");
1235 // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1251 // TODO: Is register only support adequate?
1266 // moving is postponed until after all the registers are stashed so nothing
1267 // is clobbers. We've already added nops to account for the size of mov and
1268 // push if the register is in the right place, so we only have to worry about
1270 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1278 // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1289 // Restore caller-saved and used registers.
1290 for (unsigned I = sizeof UsedMask; I-- > 0;)
1296 OutStreamer->AddComment("xray typed event end.");
1307 const Function &F = MF->getFunction();
1308 if (F.hasFnAttribute("patchable-function-entry")) {
1310 if (F.getFnAttribute("patchable-function-entry")
1327 // mov %r10, <function id, 32-bit> // 6 bytes
1328 // call <relative offset, 32-bits> // 5 bytes
1331 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1332 OutStreamer->emitLabel(CurSled);
1334 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1337 OutStreamer->emitBytes("\xeb\x09");
1359 // This just makes sure that the alignment for the next instruction is 2.
1361 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1362 OutStreamer->emitLabel(CurSled);
1369 OutStreamer->emitInstruction(Ret, getSubtargetInfo());
1414 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1415 OutStreamer->emitLabel(CurSled);
1418 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1421 OutStreamer->emitBytes("\xeb\x09");
1423 OutStreamer->emitLabel(Target);
1426 // Before emitting the instruction, add a comment to indicate that this is
1428 OutStreamer->AddComment("TAILCALL");
1432 OutStreamer->emitInstruction(TC, getSubtargetInfo());
1435 OutStreamer->emitLabel(FallthroughLabel);
1439 // If MBBI is the first instruction of the first basic block, returns null.
1442 const MachineBasicBlock *MBB = MBBI->getParent();
1443 while (MBBI == MBB->begin()) {
1444 if (MBB == &MBB->getParent()->front())
1446 MBB = MBB->getPrevNode();
1447 MBBI = MBB->end();
1449 --MBBI;
1454 if (X86II::isKMasked(MI->getDesc().TSFlags)) {
1457 if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
1465 static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI,
1467 const MachineOperand &DstOp = MI->getOperand(0);
1468 CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg());
1473 if (X86II::isKMasked(MI->getDesc().TSFlags)) {
1474 const MachineOperand &WriteMaskOp = MI->getOperand(SrcOpIdx - 1);
1476 CS << " {%" << Mask << "}";
1477 if (!X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
1478 CS << " {z}";
1483 static void printShuffleMask(raw_ostream &CS, StringRef Src1Name,
1490 ShuffleMask[i] -= e;
1494 CS << ",";
1496 CS << "zero";
1503 CS << (isSrc1 ? Src1Name : Src2Name) << '[';
1509 CS << ',';
1513 CS << "u";
1515 CS << ShuffleMask[i] % (int)e;
1518 CS << ']';
1519 --i; // For loop increments element #.
1527 const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
1528 const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
1536 raw_string_ostream CS(Comment);
1537 printDstRegisterName(CS, MI, SrcOp1Idx);
1538 CS << " = ";
1539 printShuffleMask(CS, Src1Name, Src2Name, Mask);
1540 CS.flush();
1545 static void printConstant(const APInt &Val, raw_ostream &CS,
1548 CS << (PrintZero ? 0ULL : Val.getZExtValue());
1550 // print multi-word constant as (w0,w1)
1551 CS << "(";
1554 CS << ",";
1555 CS << (PrintZero ? 0ULL : Val.getRawData()[i]);
1557 CS << ")";
1561 static void printConstant(const APFloat &Flt, raw_ostream &CS,
1569 CS << Str;
1573 raw_ostream &CS, bool PrintZero = false) {
1575 CS << "u";
1577 printConstant(CI->getValue(), CS, PrintZero);
1579 printConstant(CF->getValueAPF(), CS, PrintZero);
1581 Type *EltTy = CDS->getElementType();
1582 bool IsInteger = EltTy->isIntegerTy();
1583 bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
1584 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
1585 unsigned E = std::min(BitWidth / EltBits, CDS->getNumElements());
1589 CS << ",";
1591 printConstant(CDS->getElementAsAPInt(I), CS, PrintZero);
1593 printConstant(CDS->getElementAsAPFloat(I), CS, PrintZero);
1595 CS << "?";
1598 unsigned EltBits = CV->getType()->getScalarSizeInBits();
1599 unsigned E = std::min(BitWidth / EltBits, CV->getNumOperands());
1603 CS << ",";
1604 printConstant(CV->getOperand(I), EltBits, CS, PrintZero);
1607 CS << "?";
1617 raw_string_ostream CS(Comment);
1618 printDstRegisterName(CS, MI, SrcIdx);
1619 CS << " = ";
1622 CS << "[";
1623 printConstant(C, SclWidth, CS);
1625 CS << ",";
1626 printConstant(C, SclWidth, CS, true);
1628 CS << "]";
1629 OutStreamer.AddComment(CS.str());
1630 return; // early-out
1634 CS << ShuffleComment;
1635 OutStreamer.AddComment(CS.str());
1643 raw_string_ostream CS(Comment);
1644 printDstRegisterName(CS, MI, SrcIdx);
1645 CS << " = [";
1648 CS << ",";
1649 printConstant(C, BitWidth, CS);
1651 CS << "]";
1652 OutStreamer.AddComment(CS.str());
1660 if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) {
1662 int NumElts = CDS->getNumElements();
1664 raw_string_ostream CS(Comment);
1665 printDstRegisterName(CS, MI, SrcIdx);
1666 CS << " = [";
1669 CS << ",";
1670 if (CDS->getElementType()->isIntegerTy()) {
1671 APInt Elt = CDS->getElementAsAPInt(i);
1673 printConstant(Elt, CS);
1675 CS << "?";
1677 CS << "]";
1678 OutStreamer.AddComment(CS.str());
1696 raw_string_ostream CS(Comment);
1697 printDstRegisterName(CS, MI, getSrcIdx(MI, 1));
1698 CS << " = ";
1701 unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1705 printShuffleMask(CS, "mem", "", Mask);
1707 OutStreamer.AddComment(CS.str());
1711 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
1715 // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
1718 static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
1719 switch (MI->getOpcode()) {
1721 XTS->emitFPOPushReg(MI->getOperand(0).getImm());
1724 XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());
1727 XTS->emitFPOStackAlign(MI->getOperand(0).getImm());
1730 assert(MI->getOperand(1).getImm() == 0 &&
1732 XTS->emitFPOSetFrame(MI->getOperand(0).getImm());
1735 XTS->emitFPOEndPrologue();
1749 switch (MI->getOpcode()) {
1751 OutStreamer->emitWinCFIPushReg(MI->getOperand(0).getImm());
1755 OutStreamer->emitWinCFISaveReg(MI->getOperand(0).getImm(),
1756 MI->getOperand(1).getImm());
1760 OutStreamer->emitWinCFISaveXMM(MI->getOperand(0).getImm(),
1761 MI->getOperand(1).getImm());
1765 OutStreamer->emitWinCFIAllocStack(MI->getOperand(0).getImm());
1769 OutStreamer->emitWinCFISetFrame(MI->getOperand(0).getImm(),
1770 MI->getOperand(1).getImm());
1774 OutStreamer->emitWinCFIPushFrame(MI->getOperand(0).getImm());
1778 OutStreamer->emitWinCFIEndProlog();
1788 switch (MI->getOpcode()) {
1806 unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1828 unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1849 unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1862 assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
1865 const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
1870 switch (MI->getOpcode()) {
1877 unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1888 unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1900 raw_string_ostream CS(Comment);
1901 const MachineOperand &DstOp = MI->getOperand(0);
1902 CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
1904 CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);
1905 OutStreamer.AddComment(CS.str());
1916 INSTR_CASE(V, Instr, , ) /* AVX-128 */ \
1917 INSTR_CASE(V, Instr, Y, ) /* AVX-256 */ \
1932 if (C->getType()->getScalarSizeInBits() == 8) {
1934 raw_string_ostream CS(Comment);
1936 X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1937 CS << "[";
1938 printConstant(C, VectorWidth, CS);
1939 CS << "]";
1940 OutStreamer.AddComment(CS.str());
1953 if (C->getType()->getScalarSizeInBits() == 16) {
1955 raw_string_ostream CS(Comment);
1957 X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);
1958 CS << "[";
1959 printConstant(C, VectorWidth, CS);
1960 CS << "]";
1961 OutStreamer.AddComment(CS.str());
2024 MOV_CASE(V, ) /* AVX-128 */ \
2030 MOV_CASE(V, Y) /* AVX-256 */ \
2189 // X86_MC::verifyInstructionPredicates(MI->getOpcode(),
2190 // Subtarget->getFeatureBits());
2194 MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2196 if (MI->getOpcode() == X86::OR64rm) {
2197 for (auto &Opd : MI->operands()) {
2206 if (OutStreamer->isVerboseAsm())
2211 if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY)
2212 OutStreamer->AddComment("EVEX TO LEGACY Compression ", false);
2213 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2214 OutStreamer->AddComment("EVEX TO VEX Compression ", false);
2215 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX)
2216 OutStreamer->AddComment("EVEX TO EVEX Compression ", false);
2219 switch (MI->getOpcode()) {
2226 Register Reg = MI->getOperand(0).getReg();
2227 OutStreamer->AddComment(StringRef("eh_return, addr: %") +
2233 OutStreamer->AddComment("CLEANUPRET");
2239 OutStreamer->AddComment("CATCHRET");
2246 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2247 // non-empty. If MI is the initial ENDBR, place the
2251 MI == &MF->front().front()) {
2256 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
2263 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))
2276 OutStreamer->AddComment("TAILCALL");
2290 // This is a pseudo op for a two instruction sequence with a label, which
2297 MCSymbol *PICBase = MF->getPICBaseSymbol();
2305 MF->getSubtarget<X86Subtarget>().getFrameLowering();
2306 bool hasFP = FrameLowering->hasFP(*MF);
2308 // TODO: This is needed only if we require precise CFA.
2309 bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2310 !OutStreamer->getDwarfFrameInfos().back().End;
2312 int stackGrowth = -RI->getSlotSize();
2315 OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);
2316 MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
2320 OutStreamer->emitLabel(PICBase);
2324 MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));
2327 OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);
2334 if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2341 // MYGLOBAL + (. - PICBASE)
2345 OutStreamer->emitLabel(DotSym);
2348 MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
2352 MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);
2359 .addReg(MI->getOperand(0).getReg())
2360 .addReg(MI->getOperand(1).getReg())
2426 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2435 if (MBBI->isCall() || !MBBI->isPseudo()) {
2436 if (MBBI->isCall())
2449 .addImm(MI->getOperand(0).getImm())
2453 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))
2457 // Two instruction prefixes (2EH for branch not-taken and 3EH for branch
2463 MachineBasicBlock *DestBB = MI->getOperand(0).getMBB();
2465 MBPI->getEdgeProbability(MI->getParent(), DestBB);
2478 // in to the stackmap shadow. The only way to achieve this is if the call
2479 // is at the end of the shadow.
2480 if (MI->isCall()) {
2487 OutStreamer->emitInstruction(TmpInst, getSubtargetInfo());