1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/Casting.h"
28 #include "llvm/Support/ErrorHandling.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include <cassert>
31 #include <cstdint>
32 #include <cstdlib>
33
34 using namespace llvm;
35
36 #define DEBUG_TYPE "mccodeemitter"
37
38 namespace {
39
40 class X86MCCodeEmitter : public MCCodeEmitter {
41 const MCInstrInfo &MCII;
42 MCContext &Ctx;
43
44 public:
X86MCCodeEmitter(const MCInstrInfo & mcii,MCContext & ctx)45 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
46 : MCII(mcii), Ctx(ctx) {}
47 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
48 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
49 ~X86MCCodeEmitter() override = default;
50
51 void emitPrefix(const MCInst &MI, raw_ostream &OS,
52 const MCSubtargetInfo &STI) const override;
53
54 void encodeInstruction(const MCInst &MI, raw_ostream &OS,
55 SmallVectorImpl<MCFixup> &Fixups,
56 const MCSubtargetInfo &STI) const override;
57
58 private:
59 unsigned getX86RegNum(const MCOperand &MO) const;
60
61 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
62
63 /// \param MI a single low-level machine instruction.
64 /// \param OpNum the operand #.
65 /// \returns true if the OpNumth operand of MI require a bit to be set in
66 /// REX prefix.
67 bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
68
69 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
70 MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS,
71 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
72
73 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
74 raw_ostream &OS) const;
75
76 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
77 raw_ostream &OS) const;
78
79 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
80 uint64_t TSFlags, bool HasREX, uint64_t StartByte,
81 raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
82 const MCSubtargetInfo &STI,
83 bool ForceSIB = false) const;
84
85 bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
86 const MCSubtargetInfo &STI, raw_ostream &OS) const;
87
88 void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
89 raw_ostream &OS) const;
90
91 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
92 raw_ostream &OS) const;
93
94 bool emitOpcodePrefix(int MemOperand, const MCInst &MI,
95 const MCSubtargetInfo &STI, raw_ostream &OS) const;
96
97 bool emitREXPrefix(int MemOperand, const MCInst &MI,
98 const MCSubtargetInfo &STI, raw_ostream &OS) const;
99 };
100
101 } // end anonymous namespace
102
modRMByte(unsigned Mod,unsigned RegOpcode,unsigned RM)103 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
104 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
105 return RM | (RegOpcode << 3) | (Mod << 6);
106 }
107
emitByte(uint8_t C,raw_ostream & OS)108 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); }
109
emitConstant(uint64_t Val,unsigned Size,raw_ostream & OS)110 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
111 // Output the constant in little endian byte order.
112 for (unsigned i = 0; i != Size; ++i) {
113 emitByte(Val & 255, OS);
114 Val >>= 8;
115 }
116 }
117
118 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for
119 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset
120 /// parameter of emitImmediate.
isDispOrCDisp8(uint64_t TSFlags,int Value,int & ImmOffset)121 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
122 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
123
124 int CD8_Scale =
125 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
126 if (!HasEVEX || CD8_Scale == 0)
127 return isInt<8>(Value);
128
129 assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
130 if (Value & (CD8_Scale - 1)) // Unaligned offset
131 return false;
132
133 int CDisp8 = Value / CD8_Scale;
134 if (!isInt<8>(CDisp8))
135 return false;
136
137 // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
138 ImmOffset = CDisp8 - Value;
139 return true;
140 }
141
142 /// \returns the appropriate fixup kind to use for an immediate in an
143 /// instruction with the specified TSFlags.
getImmFixupKind(uint64_t TSFlags)144 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
145 unsigned Size = X86II::getSizeOfImm(TSFlags);
146 bool isPCRel = X86II::isImmPCRel(TSFlags);
147
148 if (X86II::isImmSigned(TSFlags)) {
149 switch (Size) {
150 default:
151 llvm_unreachable("Unsupported signed fixup size!");
152 case 4:
153 return MCFixupKind(X86::reloc_signed_4byte);
154 }
155 }
156 return MCFixup::getKindForSize(Size, isPCRel);
157 }
158
159 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
160
161 /// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is
162 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
163 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
164 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
165 /// binary expression.
166 static GlobalOffsetTableExprKind
startsWithGlobalOffsetTable(const MCExpr * Expr)167 startsWithGlobalOffsetTable(const MCExpr *Expr) {
168 const MCExpr *RHS = nullptr;
169 if (Expr->getKind() == MCExpr::Binary) {
170 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
171 Expr = BE->getLHS();
172 RHS = BE->getRHS();
173 }
174
175 if (Expr->getKind() != MCExpr::SymbolRef)
176 return GOT_None;
177
178 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
179 const MCSymbol &S = Ref->getSymbol();
180 if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
181 return GOT_None;
182 if (RHS && RHS->getKind() == MCExpr::SymbolRef)
183 return GOT_SymDiff;
184 return GOT_Normal;
185 }
186
hasSecRelSymbolRef(const MCExpr * Expr)187 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
188 if (Expr->getKind() == MCExpr::SymbolRef) {
189 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
190 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
191 }
192 return false;
193 }
194
isPCRel32Branch(const MCInst & MI,const MCInstrInfo & MCII)195 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
196 unsigned Opcode = MI.getOpcode();
197 const MCInstrDesc &Desc = MCII.get(Opcode);
198 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
199 Opcode != X86::JCC_4) ||
200 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
201 return false;
202
203 unsigned CurOp = X86II::getOperandBias(Desc);
204 const MCOperand &Op = MI.getOperand(CurOp);
205 if (!Op.isExpr())
206 return false;
207
208 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
209 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
210 }
211
getX86RegNum(const MCOperand & MO) const212 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
213 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
214 }
215
getX86RegEncoding(const MCInst & MI,unsigned OpNum) const216 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
217 unsigned OpNum) const {
218 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
219 }
220
221 /// \param MI a single low-level machine instruction.
222 /// \param OpNum the operand #.
223 /// \returns true if the OpNumth operand of MI require a bit to be set in
224 /// REX prefix.
isREXExtendedReg(const MCInst & MI,unsigned OpNum) const225 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
226 unsigned OpNum) const {
227 return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
228 }
229
emitImmediate(const MCOperand & DispOp,SMLoc Loc,unsigned Size,MCFixupKind FixupKind,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,int ImmOffset) const230 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
231 unsigned Size, MCFixupKind FixupKind,
232 uint64_t StartByte, raw_ostream &OS,
233 SmallVectorImpl<MCFixup> &Fixups,
234 int ImmOffset) const {
235 const MCExpr *Expr = nullptr;
236 if (DispOp.isImm()) {
237 // If this is a simple integer displacement that doesn't require a
238 // relocation, emit it now.
239 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
240 FixupKind != FK_PCRel_4) {
241 emitConstant(DispOp.getImm() + ImmOffset, Size, OS);
242 return;
243 }
244 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
245 } else {
246 Expr = DispOp.getExpr();
247 }
248
249 // If we have an immoffset, add it to the expression.
250 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
251 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
252 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
253 if (Kind != GOT_None) {
254 assert(ImmOffset == 0);
255
256 if (Size == 8) {
257 FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
258 } else {
259 assert(Size == 4);
260 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
261 }
262
263 if (Kind == GOT_Normal)
264 ImmOffset = static_cast<int>(OS.tell() - StartByte);
265 } else if (Expr->getKind() == MCExpr::SymbolRef) {
266 if (hasSecRelSymbolRef(Expr)) {
267 FixupKind = MCFixupKind(FK_SecRel_4);
268 }
269 } else if (Expr->getKind() == MCExpr::Binary) {
270 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
271 if (hasSecRelSymbolRef(Bin->getLHS()) ||
272 hasSecRelSymbolRef(Bin->getRHS())) {
273 FixupKind = MCFixupKind(FK_SecRel_4);
274 }
275 }
276 }
277
278 // If the fixup is pc-relative, we need to bias the value to be relative to
279 // the start of the field, not the end of the field.
280 if (FixupKind == FK_PCRel_4 ||
281 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
282 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
283 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
284 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
285 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
286 ImmOffset -= 4;
287 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
288 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
289 // this needs to be a GOTPC32 relocation.
290 if (startsWithGlobalOffsetTable(Expr) != GOT_None)
291 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
292 }
293 if (FixupKind == FK_PCRel_2)
294 ImmOffset -= 2;
295 if (FixupKind == FK_PCRel_1)
296 ImmOffset -= 1;
297
298 if (ImmOffset)
299 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
300 Ctx);
301
302 // Emit a symbolic constant as a fixup and 4 zeros.
303 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte),
304 Expr, FixupKind, Loc));
305 emitConstant(0, Size, OS);
306 }
307
emitRegModRMByte(const MCOperand & ModRMReg,unsigned RegOpcodeFld,raw_ostream & OS) const308 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
309 unsigned RegOpcodeFld,
310 raw_ostream &OS) const {
311 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS);
312 }
313
emitSIBByte(unsigned SS,unsigned Index,unsigned Base,raw_ostream & OS) const314 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
315 raw_ostream &OS) const {
316 // SIB byte is in the same format as the modRMByte.
317 emitByte(modRMByte(SS, Index, Base), OS);
318 }
319
emitMemModRMByte(const MCInst & MI,unsigned Op,unsigned RegOpcodeField,uint64_t TSFlags,bool HasREX,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI,bool ForceSIB) const320 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
321 unsigned RegOpcodeField,
322 uint64_t TSFlags, bool HasREX,
323 uint64_t StartByte, raw_ostream &OS,
324 SmallVectorImpl<MCFixup> &Fixups,
325 const MCSubtargetInfo &STI,
326 bool ForceSIB) const {
327 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
328 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
329 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
330 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
331 unsigned BaseReg = Base.getReg();
332
333 // Handle %rip relative addressing.
334 if (BaseReg == X86::RIP ||
335 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
336 assert(STI.hasFeature(X86::Is64Bit) &&
337 "Rip-relative addressing requires 64-bit mode");
338 assert(IndexReg.getReg() == 0 && !ForceSIB &&
339 "Invalid rip-relative address");
340 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
341
342 unsigned Opcode = MI.getOpcode();
343 unsigned FixupKind = [&]() {
344 // Enable relaxed relocation only for a MCSymbolRefExpr. We cannot use a
345 // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).
346 if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr())))
347 return X86::reloc_riprel_4byte;
348
349 // Certain loads for GOT references can be relocated against the symbol
350 // directly if the symbol ends up in the same linkage unit.
351 switch (Opcode) {
352 default:
353 return X86::reloc_riprel_4byte;
354 case X86::MOV64rm:
355 // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a
356 // special case because COFF and Mach-O don't support ELF's more
357 // flexible R_X86_64_REX_GOTPCRELX relaxation.
358 assert(HasREX);
359 return X86::reloc_riprel_4byte_movq_load;
360 case X86::ADC32rm:
361 case X86::ADD32rm:
362 case X86::AND32rm:
363 case X86::CMP32rm:
364 case X86::MOV32rm:
365 case X86::OR32rm:
366 case X86::SBB32rm:
367 case X86::SUB32rm:
368 case X86::TEST32mr:
369 case X86::XOR32rm:
370 case X86::CALL64m:
371 case X86::JMP64m:
372 case X86::TAILJMPm64:
373 case X86::TEST64mr:
374 case X86::ADC64rm:
375 case X86::ADD64rm:
376 case X86::AND64rm:
377 case X86::CMP64rm:
378 case X86::OR64rm:
379 case X86::SBB64rm:
380 case X86::SUB64rm:
381 case X86::XOR64rm:
382 return HasREX ? X86::reloc_riprel_4byte_relax_rex
383 : X86::reloc_riprel_4byte_relax;
384 }
385 }();
386
387 // rip-relative addressing is actually relative to the *next* instruction.
388 // Since an immediate can follow the mod/rm byte for an instruction, this
389 // means that we need to bias the displacement field of the instruction with
390 // the size of the immediate field. If we have this case, add it into the
391 // expression to emit.
392 // Note: rip-relative addressing using immediate displacement values should
393 // not be adjusted, assuming it was the user's intent.
394 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
395 ? X86II::getSizeOfImm(TSFlags)
396 : 0;
397
398 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
399 Fixups, -ImmSize);
400 return;
401 }
402
403 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
404
405 // 16-bit addressing forms of the ModR/M byte have a different encoding for
406 // the R/M field and are far more limited in which registers can be used.
407 if (X86_MC::is16BitMemOperand(MI, Op, STI)) {
408 if (BaseReg) {
409 // For 32-bit addressing, the row and column values in Table 2-2 are
410 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
411 // some special cases. And getX86RegNum reflects that numbering.
412 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
413 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
414 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
415 // while values 0-3 indicate the allowed combinations (base+index) of
416 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
417 //
418 // R16Table[] is a lookup from the normal RegNo, to the row values from
419 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
420 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
421 unsigned RMfield = R16Table[BaseRegNo];
422
423 assert(RMfield && "invalid 16-bit base register");
424
425 if (IndexReg.getReg()) {
426 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
427
428 assert(IndexReg16 && "invalid 16-bit index register");
429 // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
430 assert(((IndexReg16 ^ RMfield) & 2) &&
431 "invalid 16-bit base/index register combination");
432 assert(Scale.getImm() == 1 &&
433 "invalid scale for 16-bit memory reference");
434
435 // Allow base/index to appear in either order (although GAS doesn't).
436 if (IndexReg16 & 2)
437 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
438 else
439 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
440 }
441
442 if (Disp.isImm() && isInt<8>(Disp.getImm())) {
443 if (Disp.getImm() == 0 && RMfield != 6) {
444 // There is no displacement; just the register.
445 emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
446 return;
447 }
448 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
449 emitByte(modRMByte(1, RegOpcodeField, RMfield), OS);
450 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
451 return;
452 }
453 // This is the [REG]+disp16 case.
454 emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
455 } else {
456 assert(IndexReg.getReg() == 0 && "Unexpected index register!");
457 // There is no BaseReg; this is the plain [disp16] case.
458 emitByte(modRMByte(0, RegOpcodeField, 6), OS);
459 }
460
461 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
462 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups);
463 return;
464 }
465
466 // Check for presence of {disp8} or {disp32} pseudo prefixes.
467 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
468 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
469
470 // We only allow no displacement if no pseudo prefix is present.
471 bool AllowNoDisp = !UseDisp8 && !UseDisp32;
472 // Disp8 is allowed unless the {disp32} prefix is present.
473 bool AllowDisp8 = !UseDisp32;
474
475 // Determine whether a SIB byte is needed.
476 if (// The SIB byte must be used if there is an index register or the
477 // encoding requires a SIB byte.
478 !ForceSIB && IndexReg.getReg() == 0 &&
479 // The SIB byte must be used if the base is ESP/RSP/R12, all of which
480 // encode to an R/M value of 4, which indicates that a SIB byte is
481 // present.
482 BaseRegNo != N86::ESP &&
483 // If there is no base register and we're in 64-bit mode, we need a SIB
484 // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
485 (!STI.hasFeature(X86::Is64Bit) || BaseReg != 0)) {
486
487 if (BaseReg == 0) { // [disp32] in X86-32 mode
488 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
489 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups);
490 return;
491 }
492
493 // If the base is not EBP/ESP/R12/R13 and there is no displacement, use
494 // simple indirect register encoding, this handles addresses like [EAX].
495 // The encoding for [EBP] or[R13] with no displacement means [disp32] so we
496 // handle it by emitting a displacement of 0 later.
497 if (BaseRegNo != N86::EBP) {
498 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
499 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
500 return;
501 }
502
503 // If the displacement is @tlscall, treat it as a zero.
504 if (Disp.isExpr()) {
505 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
506 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
507 // This is exclusively used by call *a@tlscall(base). The relocation
508 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
509 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
510 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
511 return;
512 }
513 }
514 }
515
516 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
517 // Including a compressed disp8 for EVEX instructions that support it.
518 // This also handles the 0 displacement for [EBP] or [R13]. We can't use
519 // disp8 if the {disp32} pseudo prefix is present.
520 if (Disp.isImm() && AllowDisp8) {
521 int ImmOffset = 0;
522 if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
523 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
524 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
525 ImmOffset);
526 return;
527 }
528 }
529
530 // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
531 // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix
532 // prevented using disp8 above.
533 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
534 unsigned Opcode = MI.getOpcode();
535 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
536 : X86::reloc_signed_4byte;
537 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
538 Fixups);
539 return;
540 }
541
542 // We need a SIB byte, so start by outputting the ModR/M byte first
543 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
544 "Cannot use ESP as index reg!");
545
546 bool ForceDisp32 = false;
547 bool ForceDisp8 = false;
548 int ImmOffset = 0;
549 if (BaseReg == 0) {
550 // If there is no base register, we emit the special case SIB byte with
551 // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
552 BaseRegNo = 5;
553 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
554 ForceDisp32 = true;
555 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
556 // Base reg can't be EBP/RBP/R13 as that would end up with '5' as
557 // the base field, but that is the magic [*] nomenclature that
558 // indicates no base when mod=0. For these cases we'll emit a 0
559 // displacement instead.
560 BaseRegNo != N86::EBP) {
561 // Emit no displacement ModR/M byte
562 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
563 } else if (Disp.isImm() && AllowDisp8 &&
564 isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
565 // Displacement fits in a byte or matches an EVEX compressed disp8, use
566 // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless
567 // {disp32} pseudo prefix was used.
568 emitByte(modRMByte(1, RegOpcodeField, 4), OS);
569 ForceDisp8 = true;
570 } else {
571 // Otherwise, emit the normal disp32 encoding.
572 emitByte(modRMByte(2, RegOpcodeField, 4), OS);
573 ForceDisp32 = true;
574 }
575
576 // Calculate what the SS field value should be...
577 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
578 unsigned SS = SSTable[Scale.getImm()];
579
580 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
581
582 emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
583
584 // Do we need to output a displacement?
585 if (ForceDisp8)
586 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
587 ImmOffset);
588 else if (ForceDisp32)
589 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
590 StartByte, OS, Fixups);
591 }
592
593 /// Emit all instruction prefixes.
594 ///
595 /// \returns true if REX prefix is used, otherwise returns false.
emitPrefixImpl(unsigned & CurOp,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const596 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
597 const MCSubtargetInfo &STI,
598 raw_ostream &OS) const {
599 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
600 // Determine where the memory operand starts, if present.
601 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
602 // Emit segment override opcode prefix as needed.
603 if (MemoryOperand != -1) {
604 MemoryOperand += CurOp;
605 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS);
606 }
607
608 // Emit the repeat opcode prefix as needed.
609 unsigned Flags = MI.getFlags();
610 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
611 emitByte(0xF3, OS);
612 if (Flags & X86::IP_HAS_REPEAT_NE)
613 emitByte(0xF2, OS);
614
615 // Emit the address size opcode prefix as needed.
616 if (X86_MC::needsAddressSizeOverride(MI, STI, MemoryOperand, TSFlags) ||
617 Flags & X86::IP_HAS_AD_SIZE)
618 emitByte(0x67, OS);
619
620 uint64_t Form = TSFlags & X86II::FormMask;
621 switch (Form) {
622 default:
623 break;
624 case X86II::RawFrmDstSrc: {
625 // Emit segment override opcode prefix as needed (not for %ds).
626 if (MI.getOperand(2).getReg() != X86::DS)
627 emitSegmentOverridePrefix(2, MI, OS);
628 CurOp += 3; // Consume operands.
629 break;
630 }
631 case X86II::RawFrmSrc: {
632 // Emit segment override opcode prefix as needed (not for %ds).
633 if (MI.getOperand(1).getReg() != X86::DS)
634 emitSegmentOverridePrefix(1, MI, OS);
635 CurOp += 2; // Consume operands.
636 break;
637 }
638 case X86II::RawFrmDst: {
639 ++CurOp; // Consume operand.
640 break;
641 }
642 case X86II::RawFrmMemOffs: {
643 // Emit segment override opcode prefix as needed.
644 emitSegmentOverridePrefix(1, MI, OS);
645 break;
646 }
647 }
648
649 // REX prefix is optional, but if used must be immediately before the opcode
650 // Encoding type for this instruction.
651 uint64_t Encoding = TSFlags & X86II::EncodingMask;
652 bool HasREX = false;
653 if (Encoding)
654 emitVEXOpcodePrefix(MemoryOperand, MI, OS);
655 else
656 HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS);
657
658 return HasREX;
659 }
660
661 /// AVX instructions are encoded using a opcode prefix called VEX.
emitVEXOpcodePrefix(int MemOperand,const MCInst & MI,raw_ostream & OS) const662 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
663 raw_ostream &OS) const {
664 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
665 uint64_t TSFlags = Desc.TSFlags;
666
667 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
668
669 uint64_t Encoding = TSFlags & X86II::EncodingMask;
670 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
671 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
672 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
673
674 // VEX_R: opcode externsion equivalent to REX.R in
675 // 1's complement (inverted) form
676 //
677 // 1: Same as REX_R=0 (must be 1 in 32-bit mode)
678 // 0: Same as REX_R=1 (64 bit mode only)
679 //
680 uint8_t VEX_R = 0x1;
681 uint8_t EVEX_R2 = 0x1;
682
683 // VEX_X: equivalent to REX.X, only used when a
684 // register is used for index in SIB Byte.
685 //
686 // 1: Same as REX.X=0 (must be 1 in 32-bit mode)
687 // 0: Same as REX.X=1 (64-bit mode only)
688 uint8_t VEX_X = 0x1;
689
690 // VEX_B:
691 //
692 // 1: Same as REX_B=0 (ignored in 32-bit mode)
693 // 0: Same as REX_B=1 (64 bit mode only)
694 //
695 uint8_t VEX_B = 0x1;
696
697 // VEX_W: opcode specific (use like REX.W, or used for
698 // opcode extension, or ignored, depending on the opcode byte)
699 uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
700
701 // VEX_5M (VEX m-mmmmm field):
702 //
703 // 0b00000: Reserved for future use
704 // 0b00001: implied 0F leading opcode
705 // 0b00010: implied 0F 38 leading opcode bytes
706 // 0b00011: implied 0F 3A leading opcode bytes
707 // 0b00100: Reserved for future use
708 // 0b00101: VEX MAP5
709 // 0b00110: VEX MAP6
710 // 0b00111-0b11111: Reserved for future use
711 // 0b01000: XOP map select - 08h instructions with imm byte
712 // 0b01001: XOP map select - 09h instructions with no imm byte
713 // 0b01010: XOP map select - 0Ah instructions with imm dword
714 uint8_t VEX_5M;
715 switch (TSFlags & X86II::OpMapMask) {
716 default:
717 llvm_unreachable("Invalid prefix!");
718 case X86II::TB:
719 VEX_5M = 0x1;
720 break; // 0F
721 case X86II::T8:
722 VEX_5M = 0x2;
723 break; // 0F 38
724 case X86II::TA:
725 VEX_5M = 0x3;
726 break; // 0F 3A
727 case X86II::XOP8:
728 VEX_5M = 0x8;
729 break;
730 case X86II::XOP9:
731 VEX_5M = 0x9;
732 break;
733 case X86II::XOPA:
734 VEX_5M = 0xA;
735 break;
736 case X86II::T_MAP5:
737 VEX_5M = 0x5;
738 break;
739 case X86II::T_MAP6:
740 VEX_5M = 0x6;
741 break;
742 }
743
744 // VEX_4V (VEX vvvv field): a register specifier
745 // (in 1's complement form) or 1111 if unused.
746 uint8_t VEX_4V = 0xf;
747 uint8_t EVEX_V2 = 0x1;
748
749 // EVEX_L2/VEX_L (Vector Length):
750 //
751 // L2 L
752 // 0 0: scalar or 128-bit vector
753 // 0 1: 256-bit vector
754 // 1 0: 512-bit vector
755 //
756 uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
757 uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
758
759 // VEX_PP: opcode extension providing equivalent
760 // functionality of a SIMD prefix
761 //
762 // 0b00: None
763 // 0b01: 66
764 // 0b10: F3
765 // 0b11: F2
766 //
767 uint8_t VEX_PP = 0;
768 switch (TSFlags & X86II::OpPrefixMask) {
769 case X86II::PD:
770 VEX_PP = 0x1;
771 break; // 66
772 case X86II::XS:
773 VEX_PP = 0x2;
774 break; // F3
775 case X86II::XD:
776 VEX_PP = 0x3;
777 break; // F2
778 }
779
780 // EVEX_U
781 uint8_t EVEX_U = 1; // Always '1' so far
782
783 // EVEX_z
784 uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
785
786 // EVEX_b
787 uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
788
789 // EVEX_rc
790 uint8_t EVEX_rc = 0;
791
792 // EVEX_aaa
793 uint8_t EVEX_aaa = 0;
794
795 bool EncodeRC = false;
796
797 // Classify VEX_B, VEX_4V, VEX_R, VEX_X
798 unsigned NumOps = Desc.getNumOperands();
799 unsigned CurOp = X86II::getOperandBias(Desc);
800
801 switch (TSFlags & X86II::FormMask) {
802 default:
803 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
804 case X86II::MRMDestMem4VOp3CC: {
805 // MemAddr, src1(ModR/M), src2(VEX_4V)
806 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
807 VEX_B = ~(BaseRegEnc >> 3) & 1;
808 unsigned IndexRegEnc =
809 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
810 VEX_X = ~(IndexRegEnc >> 3) & 1;
811
812 CurOp += X86::AddrNumOperands;
813
814 unsigned RegEnc = getX86RegEncoding(MI, ++CurOp);
815 VEX_R = ~(RegEnc >> 3) & 1;
816
817 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
818 VEX_4V = ~VRegEnc & 0xf;
819 break;
820 }
821 case X86II::MRM_C0:
822 case X86II::RawFrm:
823 case X86II::PrefixByte:
824 break;
825 case X86II::MRMDestMemFSIB:
826 case X86II::MRMDestMem: {
827 // MRMDestMem instructions forms:
828 // MemAddr, src1(ModR/M)
829 // MemAddr, src1(VEX_4V), src2(ModR/M)
830 // MemAddr, src1(ModR/M), imm8
831 //
832 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
833 VEX_B = ~(BaseRegEnc >> 3) & 1;
834 unsigned IndexRegEnc =
835 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
836 VEX_X = ~(IndexRegEnc >> 3) & 1;
837 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
838 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
839
840 CurOp += X86::AddrNumOperands;
841
842 if (HasEVEX_K)
843 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
844
845 if (HasVEX_4V) {
846 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
847 VEX_4V = ~VRegEnc & 0xf;
848 EVEX_V2 = ~(VRegEnc >> 4) & 1;
849 }
850
851 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
852 VEX_R = ~(RegEnc >> 3) & 1;
853 EVEX_R2 = ~(RegEnc >> 4) & 1;
854 break;
855 }
856 case X86II::MRMSrcMemFSIB:
857 case X86II::MRMSrcMem: {
858 // MRMSrcMem instructions forms:
859 // src1(ModR/M), MemAddr
860 // src1(ModR/M), src2(VEX_4V), MemAddr
861 // src1(ModR/M), MemAddr, imm8
862 // src1(ModR/M), MemAddr, src2(Imm[7:4])
863 //
864 // FMA4:
865 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
866 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
867 VEX_R = ~(RegEnc >> 3) & 1;
868 EVEX_R2 = ~(RegEnc >> 4) & 1;
869
870 if (HasEVEX_K)
871 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
872
873 if (HasVEX_4V) {
874 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
875 VEX_4V = ~VRegEnc & 0xf;
876 EVEX_V2 = ~(VRegEnc >> 4) & 1;
877 }
878
879 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
880 VEX_B = ~(BaseRegEnc >> 3) & 1;
881 unsigned IndexRegEnc =
882 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
883 VEX_X = ~(IndexRegEnc >> 3) & 1;
884 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
885 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
886
887 break;
888 }
889 case X86II::MRMSrcMem4VOp3: {
890 // Instruction format for 4VOp3:
891 // src1(ModR/M), MemAddr, src3(VEX_4V)
892 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
893 VEX_R = ~(RegEnc >> 3) & 1;
894
895 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
896 VEX_B = ~(BaseRegEnc >> 3) & 1;
897 unsigned IndexRegEnc =
898 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
899 VEX_X = ~(IndexRegEnc >> 3) & 1;
900
901 VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
902 break;
903 }
904 case X86II::MRMSrcMemOp4: {
905 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
906 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
907 VEX_R = ~(RegEnc >> 3) & 1;
908
909 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
910 VEX_4V = ~VRegEnc & 0xf;
911
912 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
913 VEX_B = ~(BaseRegEnc >> 3) & 1;
914 unsigned IndexRegEnc =
915 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
916 VEX_X = ~(IndexRegEnc >> 3) & 1;
917 break;
918 }
919 case X86II::MRM0m:
920 case X86II::MRM1m:
921 case X86II::MRM2m:
922 case X86II::MRM3m:
923 case X86II::MRM4m:
924 case X86II::MRM5m:
925 case X86II::MRM6m:
926 case X86II::MRM7m: {
927 // MRM[0-9]m instructions forms:
928 // MemAddr
929 // src1(VEX_4V), MemAddr
930 if (HasVEX_4V) {
931 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
932 VEX_4V = ~VRegEnc & 0xf;
933 EVEX_V2 = ~(VRegEnc >> 4) & 1;
934 }
935
936 if (HasEVEX_K)
937 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
938
939 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
940 VEX_B = ~(BaseRegEnc >> 3) & 1;
941 unsigned IndexRegEnc =
942 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
943 VEX_X = ~(IndexRegEnc >> 3) & 1;
944 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
945 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
946
947 break;
948 }
949 case X86II::MRMSrcReg: {
950 // MRMSrcReg instructions forms:
951 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
952 // dst(ModR/M), src1(ModR/M)
953 // dst(ModR/M), src1(ModR/M), imm8
954 //
955 // FMA4:
956 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
957 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
958 VEX_R = ~(RegEnc >> 3) & 1;
959 EVEX_R2 = ~(RegEnc >> 4) & 1;
960
961 if (HasEVEX_K)
962 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
963
964 if (HasVEX_4V) {
965 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
966 VEX_4V = ~VRegEnc & 0xf;
967 EVEX_V2 = ~(VRegEnc >> 4) & 1;
968 }
969
970 RegEnc = getX86RegEncoding(MI, CurOp++);
971 VEX_B = ~(RegEnc >> 3) & 1;
972 VEX_X = ~(RegEnc >> 4) & 1;
973
974 if (EVEX_b) {
975 if (HasEVEX_RC) {
976 unsigned RcOperand = NumOps - 1;
977 assert(RcOperand >= CurOp);
978 EVEX_rc = MI.getOperand(RcOperand).getImm();
979 assert(EVEX_rc <= 3 && "Invalid rounding control!");
980 }
981 EncodeRC = true;
982 }
983 break;
984 }
985 case X86II::MRMSrcReg4VOp3: {
986 // Instruction format for 4VOp3:
987 // src1(ModR/M), src2(ModR/M), src3(VEX_4V)
988 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
989 VEX_R = ~(RegEnc >> 3) & 1;
990
991 RegEnc = getX86RegEncoding(MI, CurOp++);
992 VEX_B = ~(RegEnc >> 3) & 1;
993
994 VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
995 break;
996 }
997 case X86II::MRMSrcRegOp4: {
998 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
999 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1000 VEX_R = ~(RegEnc >> 3) & 1;
1001
1002 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1003 VEX_4V = ~VRegEnc & 0xf;
1004
1005 // Skip second register source (encoded in Imm[7:4])
1006 ++CurOp;
1007
1008 RegEnc = getX86RegEncoding(MI, CurOp++);
1009 VEX_B = ~(RegEnc >> 3) & 1;
1010 VEX_X = ~(RegEnc >> 4) & 1;
1011 break;
1012 }
1013 case X86II::MRMDestReg: {
1014 // MRMDestReg instructions forms:
1015 // dst(ModR/M), src(ModR/M)
1016 // dst(ModR/M), src(ModR/M), imm8
1017 // dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1018 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1019 VEX_B = ~(RegEnc >> 3) & 1;
1020 VEX_X = ~(RegEnc >> 4) & 1;
1021
1022 if (HasEVEX_K)
1023 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1024
1025 if (HasVEX_4V) {
1026 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1027 VEX_4V = ~VRegEnc & 0xf;
1028 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1029 }
1030
1031 RegEnc = getX86RegEncoding(MI, CurOp++);
1032 VEX_R = ~(RegEnc >> 3) & 1;
1033 EVEX_R2 = ~(RegEnc >> 4) & 1;
1034 if (EVEX_b)
1035 EncodeRC = true;
1036 break;
1037 }
1038 case X86II::MRMr0: {
1039 // MRMr0 instructions forms:
1040 // 11:rrr:000
1041 // dst(ModR/M)
1042 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1043 VEX_R = ~(RegEnc >> 3) & 1;
1044 EVEX_R2 = ~(RegEnc >> 4) & 1;
1045 break;
1046 }
1047 case X86II::MRM0r:
1048 case X86II::MRM1r:
1049 case X86II::MRM2r:
1050 case X86II::MRM3r:
1051 case X86II::MRM4r:
1052 case X86II::MRM5r:
1053 case X86II::MRM6r:
1054 case X86II::MRM7r: {
1055 // MRM0r-MRM7r instructions forms:
1056 // dst(VEX_4V), src(ModR/M), imm8
1057 if (HasVEX_4V) {
1058 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1059 VEX_4V = ~VRegEnc & 0xf;
1060 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1061 }
1062 if (HasEVEX_K)
1063 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1064
1065 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1066 VEX_B = ~(RegEnc >> 3) & 1;
1067 VEX_X = ~(RegEnc >> 4) & 1;
1068 break;
1069 }
1070 }
1071
1072 if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1073 // VEX opcode prefix can have 2 or 3 bytes
1074 //
1075 // 3 bytes:
1076 // +-----+ +--------------+ +-------------------+
1077 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1078 // +-----+ +--------------+ +-------------------+
1079 // 2 bytes:
1080 // +-----+ +-------------------+
1081 // | C5h | | R | vvvv | L | pp |
1082 // +-----+ +-------------------+
1083 //
1084 // XOP uses a similar prefix:
1085 // +-----+ +--------------+ +-------------------+
1086 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1087 // +-----+ +--------------+ +-------------------+
1088 uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1089
1090 // Can we use the 2 byte VEX prefix?
1091 if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1092 VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1093 emitByte(0xC5, OS);
1094 emitByte(LastByte | (VEX_R << 7), OS);
1095 return;
1096 }
1097
1098 // 3 byte VEX prefix
1099 emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS);
1100 emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS);
1101 emitByte(LastByte | (VEX_W << 7), OS);
1102 } else {
1103 assert(Encoding == X86II::EVEX && "unknown encoding!");
1104 // EVEX opcode prefix can have 4 bytes
1105 //
1106 // +-----+ +--------------+ +-------------------+ +------------------------+
1107 // | 62h | | RXBR' | 0mmm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1108 // +-----+ +--------------+ +-------------------+ +------------------------+
1109 assert((VEX_5M & 0x7) == VEX_5M &&
1110 "More than 3 significant bits in VEX.m-mmmm fields for EVEX!");
1111
1112 emitByte(0x62, OS);
1113 emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1114 VEX_5M,
1115 OS);
1116 emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS);
1117 if (EncodeRC)
1118 emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1119 EVEX_aaa,
1120 OS);
1121 else
1122 emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1123 (EVEX_V2 << 3) | EVEX_aaa,
1124 OS);
1125 }
1126 }
1127
1128 /// Emit REX prefix which specifies
1129 /// 1) 64-bit instructions,
1130 /// 2) non-default operand size, and
1131 /// 3) use of X86-64 extended registers.
1132 ///
1133 /// \returns true if REX prefix is used, otherwise returns false.
emitREXPrefix(int MemOperand,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const1134 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1135 const MCSubtargetInfo &STI,
1136 raw_ostream &OS) const {
1137 uint8_t REX = [&, MemOperand]() {
1138 uint8_t REX = 0;
1139 bool UsesHighByteReg = false;
1140
1141 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1142 uint64_t TSFlags = Desc.TSFlags;
1143
1144 if (TSFlags & X86II::REX_W)
1145 REX |= 1 << 3; // set REX.W
1146
1147 if (MI.getNumOperands() == 0)
1148 return REX;
1149
1150 unsigned NumOps = MI.getNumOperands();
1151 unsigned CurOp = X86II::getOperandBias(Desc);
1152
1153 // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1154 for (unsigned i = CurOp; i != NumOps; ++i) {
1155 const MCOperand &MO = MI.getOperand(i);
1156 if (MO.isReg()) {
1157 unsigned Reg = MO.getReg();
1158 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH ||
1159 Reg == X86::DH)
1160 UsesHighByteReg = true;
1161 if (X86II::isX86_64NonExtLowByteReg(Reg))
1162 // FIXME: The caller of determineREXPrefix slaps this prefix onto
1163 // anything that returns non-zero.
1164 REX |= 0x40; // REX fixed encoding prefix
1165 } else if (MO.isExpr() && STI.getTargetTriple().isX32()) {
1166 // GOTTPOFF and TLSDESC relocations require a REX prefix to allow
1167 // linker optimizations: even if the instructions we see may not require
1168 // any prefix, they may be replaced by instructions that do. This is
1169 // handled as a special case here so that it also works for hand-written
1170 // assembly without the user needing to write REX, as with GNU as.
1171 const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());
1172 if (Ref && (Ref->getKind() == MCSymbolRefExpr::VK_GOTTPOFF ||
1173 Ref->getKind() == MCSymbolRefExpr::VK_TLSDESC)) {
1174 REX |= 0x40; // REX fixed encoding prefix
1175 }
1176 }
1177 }
1178
1179 switch (TSFlags & X86II::FormMask) {
1180 case X86II::AddRegFrm:
1181 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1182 break;
1183 case X86II::MRMSrcReg:
1184 case X86II::MRMSrcRegCC:
1185 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1186 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1187 break;
1188 case X86II::MRMSrcMem:
1189 case X86II::MRMSrcMemCC:
1190 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1191 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1192 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1193 CurOp += X86::AddrNumOperands;
1194 break;
1195 case X86II::MRMDestReg:
1196 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1197 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1198 break;
1199 case X86II::MRMDestMem:
1200 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1201 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1202 CurOp += X86::AddrNumOperands;
1203 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1204 break;
1205 case X86II::MRMXmCC:
1206 case X86II::MRMXm:
1207 case X86II::MRM0m:
1208 case X86II::MRM1m:
1209 case X86II::MRM2m:
1210 case X86II::MRM3m:
1211 case X86II::MRM4m:
1212 case X86II::MRM5m:
1213 case X86II::MRM6m:
1214 case X86II::MRM7m:
1215 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1216 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1217 break;
1218 case X86II::MRMXrCC:
1219 case X86II::MRMXr:
1220 case X86II::MRM0r:
1221 case X86II::MRM1r:
1222 case X86II::MRM2r:
1223 case X86II::MRM3r:
1224 case X86II::MRM4r:
1225 case X86II::MRM5r:
1226 case X86II::MRM6r:
1227 case X86II::MRM7r:
1228 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1229 break;
1230 case X86II::MRMr0:
1231 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1232 break;
1233 case X86II::MRMDestMemFSIB:
1234 llvm_unreachable("FSIB format never need REX prefix!");
1235 }
1236 if (REX && UsesHighByteReg)
1237 report_fatal_error(
1238 "Cannot encode high byte register in REX-prefixed instruction");
1239 return REX;
1240 }();
1241
1242 if (!REX)
1243 return false;
1244
1245 emitByte(0x40 | REX, OS);
1246 return true;
1247 }
1248
1249 /// Emit segment override opcode prefix as needed.
emitSegmentOverridePrefix(unsigned SegOperand,const MCInst & MI,raw_ostream & OS) const1250 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand,
1251 const MCInst &MI,
1252 raw_ostream &OS) const {
1253 // Check for explicit segment override on memory operand.
1254 if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1255 emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS);
1256 }
1257
1258 /// Emit all instruction prefixes prior to the opcode.
1259 ///
1260 /// \param MemOperand the operand # of the start of a memory operand if present.
1261 /// If not present, it is -1.
1262 ///
1263 /// \returns true if REX prefix is used, otherwise returns false.
emitOpcodePrefix(int MemOperand,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const1264 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1265 const MCSubtargetInfo &STI,
1266 raw_ostream &OS) const {
1267 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1268 uint64_t TSFlags = Desc.TSFlags;
1269
1270 // Emit the operand size opcode prefix as needed.
1271 if ((TSFlags & X86II::OpSizeMask) ==
1272 (STI.hasFeature(X86::Is16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1273 emitByte(0x66, OS);
1274
1275 // Emit the LOCK opcode prefix.
1276 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1277 emitByte(0xF0, OS);
1278
1279 // Emit the NOTRACK opcode prefix.
1280 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1281 emitByte(0x3E, OS);
1282
1283 switch (TSFlags & X86II::OpPrefixMask) {
1284 case X86II::PD: // 66
1285 emitByte(0x66, OS);
1286 break;
1287 case X86II::XS: // F3
1288 emitByte(0xF3, OS);
1289 break;
1290 case X86II::XD: // F2
1291 emitByte(0xF2, OS);
1292 break;
1293 }
1294
1295 // Handle REX prefix.
1296 assert((STI.hasFeature(X86::Is64Bit) || !(TSFlags & X86II::REX_W)) &&
1297 "REX.W requires 64bit mode.");
1298 bool HasREX = STI.hasFeature(X86::Is64Bit)
1299 ? emitREXPrefix(MemOperand, MI, STI, OS)
1300 : false;
1301
1302 // 0x0F escape code must be emitted just before the opcode.
1303 switch (TSFlags & X86II::OpMapMask) {
1304 case X86II::TB: // Two-byte opcode map
1305 case X86II::T8: // 0F 38
1306 case X86II::TA: // 0F 3A
1307 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1308 emitByte(0x0F, OS);
1309 break;
1310 }
1311
1312 switch (TSFlags & X86II::OpMapMask) {
1313 case X86II::T8: // 0F 38
1314 emitByte(0x38, OS);
1315 break;
1316 case X86II::TA: // 0F 3A
1317 emitByte(0x3A, OS);
1318 break;
1319 }
1320
1321 return HasREX;
1322 }
1323
emitPrefix(const MCInst & MI,raw_ostream & OS,const MCSubtargetInfo & STI) const1324 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1325 const MCSubtargetInfo &STI) const {
1326 unsigned Opcode = MI.getOpcode();
1327 const MCInstrDesc &Desc = MCII.get(Opcode);
1328 uint64_t TSFlags = Desc.TSFlags;
1329
1330 // Pseudo instructions don't get encoded.
1331 if (X86II::isPseudo(TSFlags))
1332 return;
1333
1334 unsigned CurOp = X86II::getOperandBias(Desc);
1335
1336 emitPrefixImpl(CurOp, MI, STI, OS);
1337 }
1338
encodeInstruction(const MCInst & MI,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI) const1339 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1340 SmallVectorImpl<MCFixup> &Fixups,
1341 const MCSubtargetInfo &STI) const {
1342 unsigned Opcode = MI.getOpcode();
1343 const MCInstrDesc &Desc = MCII.get(Opcode);
1344 uint64_t TSFlags = Desc.TSFlags;
1345
1346 // Pseudo instructions don't get encoded.
1347 if (X86II::isPseudo(TSFlags))
1348 return;
1349
1350 unsigned NumOps = Desc.getNumOperands();
1351 unsigned CurOp = X86II::getOperandBias(Desc);
1352
1353 uint64_t StartByte = OS.tell();
1354
1355 bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS);
1356
1357 // It uses the VEX.VVVV field?
1358 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1359 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1360
1361 // It uses the EVEX.aaa field?
1362 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1363 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1364
1365 // Used if a register is encoded in 7:4 of immediate.
1366 unsigned I8RegNum = 0;
1367
1368 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1369
1370 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1371 BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1372
1373 unsigned OpcodeOffset = 0;
1374
1375 uint64_t Form = TSFlags & X86II::FormMask;
1376 switch (Form) {
1377 default:
1378 errs() << "FORM: " << Form << "\n";
1379 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1380 case X86II::Pseudo:
1381 llvm_unreachable("Pseudo instruction shouldn't be emitted");
1382 case X86II::RawFrmDstSrc:
1383 case X86II::RawFrmSrc:
1384 case X86II::RawFrmDst:
1385 case X86II::PrefixByte:
1386 emitByte(BaseOpcode, OS);
1387 break;
1388 case X86II::AddCCFrm: {
1389 // This will be added to the opcode in the fallthrough.
1390 OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1391 assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1392 --NumOps; // Drop the operand from the end.
1393 [[fallthrough]];
1394 case X86II::RawFrm:
1395 emitByte(BaseOpcode + OpcodeOffset, OS);
1396
1397 if (!STI.hasFeature(X86::Is64Bit) || !isPCRel32Branch(MI, MCII))
1398 break;
1399
1400 const MCOperand &Op = MI.getOperand(CurOp++);
1401 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1402 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS,
1403 Fixups);
1404 break;
1405 }
1406 case X86II::RawFrmMemOffs:
1407 emitByte(BaseOpcode, OS);
1408 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1409 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1410 StartByte, OS, Fixups);
1411 ++CurOp; // skip segment operand
1412 break;
1413 case X86II::RawFrmImm8:
1414 emitByte(BaseOpcode, OS);
1415 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1416 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1417 StartByte, OS, Fixups);
1418 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1419 OS, Fixups);
1420 break;
1421 case X86II::RawFrmImm16:
1422 emitByte(BaseOpcode, OS);
1423 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1424 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1425 StartByte, OS, Fixups);
1426 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1427 OS, Fixups);
1428 break;
1429
1430 case X86II::AddRegFrm:
1431 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS);
1432 break;
1433
1434 case X86II::MRMDestReg: {
1435 emitByte(BaseOpcode, OS);
1436 unsigned SrcRegNum = CurOp + 1;
1437
1438 if (HasEVEX_K) // Skip writemask
1439 ++SrcRegNum;
1440
1441 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1442 ++SrcRegNum;
1443
1444 emitRegModRMByte(MI.getOperand(CurOp),
1445 getX86RegNum(MI.getOperand(SrcRegNum)), OS);
1446 CurOp = SrcRegNum + 1;
1447 break;
1448 }
1449 case X86II::MRMDestMem4VOp3CC: {
1450 unsigned CC = MI.getOperand(8).getImm();
1451 emitByte(BaseOpcode + CC, OS);
1452 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1453 emitMemModRMByte(MI, CurOp + 1, getX86RegNum(MI.getOperand(0)), TSFlags,
1454 HasREX, StartByte, OS, Fixups, STI, false);
1455 CurOp = SrcRegNum + 3; // skip reg, VEX_V4 and CC
1456 break;
1457 }
1458 case X86II::MRMDestMemFSIB:
1459 case X86II::MRMDestMem: {
1460 emitByte(BaseOpcode, OS);
1461 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1462
1463 if (HasEVEX_K) // Skip writemask
1464 ++SrcRegNum;
1465
1466 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1467 ++SrcRegNum;
1468
1469 bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1470 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1471 HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1472 CurOp = SrcRegNum + 1;
1473 break;
1474 }
1475 case X86II::MRMSrcReg: {
1476 emitByte(BaseOpcode, OS);
1477 unsigned SrcRegNum = CurOp + 1;
1478
1479 if (HasEVEX_K) // Skip writemask
1480 ++SrcRegNum;
1481
1482 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1483 ++SrcRegNum;
1484
1485 emitRegModRMByte(MI.getOperand(SrcRegNum),
1486 getX86RegNum(MI.getOperand(CurOp)), OS);
1487 CurOp = SrcRegNum + 1;
1488 if (HasVEX_I8Reg)
1489 I8RegNum = getX86RegEncoding(MI, CurOp++);
1490 // do not count the rounding control operand
1491 if (HasEVEX_RC)
1492 --NumOps;
1493 break;
1494 }
1495 case X86II::MRMSrcReg4VOp3: {
1496 emitByte(BaseOpcode, OS);
1497 unsigned SrcRegNum = CurOp + 1;
1498
1499 emitRegModRMByte(MI.getOperand(SrcRegNum),
1500 getX86RegNum(MI.getOperand(CurOp)), OS);
1501 CurOp = SrcRegNum + 1;
1502 ++CurOp; // Encoded in VEX.VVVV
1503 break;
1504 }
1505 case X86II::MRMSrcRegOp4: {
1506 emitByte(BaseOpcode, OS);
1507 unsigned SrcRegNum = CurOp + 1;
1508
1509 // Skip 1st src (which is encoded in VEX_VVVV)
1510 ++SrcRegNum;
1511
1512 // Capture 2nd src (which is encoded in Imm[7:4])
1513 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1514 I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1515
1516 emitRegModRMByte(MI.getOperand(SrcRegNum),
1517 getX86RegNum(MI.getOperand(CurOp)), OS);
1518 CurOp = SrcRegNum + 1;
1519 break;
1520 }
1521 case X86II::MRMSrcRegCC: {
1522 unsigned FirstOp = CurOp++;
1523 unsigned SecondOp = CurOp++;
1524
1525 unsigned CC = MI.getOperand(CurOp++).getImm();
1526 emitByte(BaseOpcode + CC, OS);
1527
1528 emitRegModRMByte(MI.getOperand(SecondOp),
1529 getX86RegNum(MI.getOperand(FirstOp)), OS);
1530 break;
1531 }
1532 case X86II::MRMSrcMemFSIB:
1533 case X86II::MRMSrcMem: {
1534 unsigned FirstMemOp = CurOp + 1;
1535
1536 if (HasEVEX_K) // Skip writemask
1537 ++FirstMemOp;
1538
1539 if (HasVEX_4V)
1540 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1541
1542 emitByte(BaseOpcode, OS);
1543
1544 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1545 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1546 TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1547 CurOp = FirstMemOp + X86::AddrNumOperands;
1548 if (HasVEX_I8Reg)
1549 I8RegNum = getX86RegEncoding(MI, CurOp++);
1550 break;
1551 }
1552 case X86II::MRMSrcMem4VOp3: {
1553 unsigned FirstMemOp = CurOp + 1;
1554
1555 emitByte(BaseOpcode, OS);
1556
1557 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1558 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1559 CurOp = FirstMemOp + X86::AddrNumOperands;
1560 ++CurOp; // Encoded in VEX.VVVV.
1561 break;
1562 }
1563 case X86II::MRMSrcMemOp4: {
1564 unsigned FirstMemOp = CurOp + 1;
1565
1566 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1567
1568 // Capture second register source (encoded in Imm[7:4])
1569 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1570 I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1571
1572 emitByte(BaseOpcode, OS);
1573
1574 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1575 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1576 CurOp = FirstMemOp + X86::AddrNumOperands;
1577 break;
1578 }
1579 case X86II::MRMSrcMemCC: {
1580 unsigned RegOp = CurOp++;
1581 unsigned FirstMemOp = CurOp;
1582 CurOp = FirstMemOp + X86::AddrNumOperands;
1583
1584 unsigned CC = MI.getOperand(CurOp++).getImm();
1585 emitByte(BaseOpcode + CC, OS);
1586
1587 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1588 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1589 break;
1590 }
1591
1592 case X86II::MRMXrCC: {
1593 unsigned RegOp = CurOp++;
1594
1595 unsigned CC = MI.getOperand(CurOp++).getImm();
1596 emitByte(BaseOpcode + CC, OS);
1597 emitRegModRMByte(MI.getOperand(RegOp), 0, OS);
1598 break;
1599 }
1600
1601 case X86II::MRMXr:
1602 case X86II::MRM0r:
1603 case X86II::MRM1r:
1604 case X86II::MRM2r:
1605 case X86II::MRM3r:
1606 case X86II::MRM4r:
1607 case X86II::MRM5r:
1608 case X86II::MRM6r:
1609 case X86II::MRM7r:
1610 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1611 ++CurOp;
1612 if (HasEVEX_K) // Skip writemask
1613 ++CurOp;
1614 emitByte(BaseOpcode, OS);
1615 emitRegModRMByte(MI.getOperand(CurOp++),
1616 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
1617 break;
1618 case X86II::MRMr0:
1619 emitByte(BaseOpcode, OS);
1620 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
1621 break;
1622
1623 case X86II::MRMXmCC: {
1624 unsigned FirstMemOp = CurOp;
1625 CurOp = FirstMemOp + X86::AddrNumOperands;
1626
1627 unsigned CC = MI.getOperand(CurOp++).getImm();
1628 emitByte(BaseOpcode + CC, OS);
1629
1630 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups,
1631 STI);
1632 break;
1633 }
1634
1635 case X86II::MRMXm:
1636 case X86II::MRM0m:
1637 case X86II::MRM1m:
1638 case X86II::MRM2m:
1639 case X86II::MRM3m:
1640 case X86II::MRM4m:
1641 case X86II::MRM5m:
1642 case X86II::MRM6m:
1643 case X86II::MRM7m:
1644 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1645 ++CurOp;
1646 if (HasEVEX_K) // Skip writemask
1647 ++CurOp;
1648 emitByte(BaseOpcode, OS);
1649 emitMemModRMByte(MI, CurOp,
1650 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1651 HasREX, StartByte, OS, Fixups, STI);
1652 CurOp += X86::AddrNumOperands;
1653 break;
1654
1655 case X86II::MRM0X:
1656 case X86II::MRM1X:
1657 case X86II::MRM2X:
1658 case X86II::MRM3X:
1659 case X86II::MRM4X:
1660 case X86II::MRM5X:
1661 case X86II::MRM6X:
1662 case X86II::MRM7X:
1663 emitByte(BaseOpcode, OS);
1664 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS);
1665 break;
1666
1667 case X86II::MRM_C0:
1668 case X86II::MRM_C1:
1669 case X86II::MRM_C2:
1670 case X86II::MRM_C3:
1671 case X86II::MRM_C4:
1672 case X86II::MRM_C5:
1673 case X86II::MRM_C6:
1674 case X86II::MRM_C7:
1675 case X86II::MRM_C8:
1676 case X86II::MRM_C9:
1677 case X86II::MRM_CA:
1678 case X86II::MRM_CB:
1679 case X86II::MRM_CC:
1680 case X86II::MRM_CD:
1681 case X86II::MRM_CE:
1682 case X86II::MRM_CF:
1683 case X86II::MRM_D0:
1684 case X86II::MRM_D1:
1685 case X86II::MRM_D2:
1686 case X86II::MRM_D3:
1687 case X86II::MRM_D4:
1688 case X86II::MRM_D5:
1689 case X86II::MRM_D6:
1690 case X86II::MRM_D7:
1691 case X86II::MRM_D8:
1692 case X86II::MRM_D9:
1693 case X86II::MRM_DA:
1694 case X86II::MRM_DB:
1695 case X86II::MRM_DC:
1696 case X86II::MRM_DD:
1697 case X86II::MRM_DE:
1698 case X86II::MRM_DF:
1699 case X86II::MRM_E0:
1700 case X86II::MRM_E1:
1701 case X86II::MRM_E2:
1702 case X86II::MRM_E3:
1703 case X86II::MRM_E4:
1704 case X86II::MRM_E5:
1705 case X86II::MRM_E6:
1706 case X86II::MRM_E7:
1707 case X86II::MRM_E8:
1708 case X86II::MRM_E9:
1709 case X86II::MRM_EA:
1710 case X86II::MRM_EB:
1711 case X86II::MRM_EC:
1712 case X86II::MRM_ED:
1713 case X86II::MRM_EE:
1714 case X86II::MRM_EF:
1715 case X86II::MRM_F0:
1716 case X86II::MRM_F1:
1717 case X86II::MRM_F2:
1718 case X86II::MRM_F3:
1719 case X86II::MRM_F4:
1720 case X86II::MRM_F5:
1721 case X86II::MRM_F6:
1722 case X86II::MRM_F7:
1723 case X86II::MRM_F8:
1724 case X86II::MRM_F9:
1725 case X86II::MRM_FA:
1726 case X86II::MRM_FB:
1727 case X86II::MRM_FC:
1728 case X86II::MRM_FD:
1729 case X86II::MRM_FE:
1730 case X86II::MRM_FF:
1731 emitByte(BaseOpcode, OS);
1732 emitByte(0xC0 + Form - X86II::MRM_C0, OS);
1733 break;
1734 }
1735
1736 if (HasVEX_I8Reg) {
1737 // The last source register of a 4 operand instruction in AVX is encoded
1738 // in bits[7:4] of a immediate byte.
1739 assert(I8RegNum < 16 && "Register encoding out of range");
1740 I8RegNum <<= 4;
1741 if (CurOp != NumOps) {
1742 unsigned Val = MI.getOperand(CurOp++).getImm();
1743 assert(Val < 16 && "Immediate operand value out of range");
1744 I8RegNum |= Val;
1745 }
1746 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1747 StartByte, OS, Fixups);
1748 } else {
1749 // If there is a remaining operand, it must be a trailing immediate. Emit it
1750 // according to the right size for the instruction. Some instructions
1751 // (SSE4a extrq and insertq) have two trailing immediates.
1752 while (CurOp != NumOps && NumOps - CurOp <= 2) {
1753 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1754 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1755 StartByte, OS, Fixups);
1756 }
1757 }
1758
1759 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1760 emitByte(X86II::getBaseOpcodeFor(TSFlags), OS);
1761
1762 assert(OS.tell() - StartByte <= 15 &&
1763 "The size of instruction must be no longer than 15.");
1764 #ifndef NDEBUG
1765 // FIXME: Verify.
1766 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1767 errs() << "Cannot encode all operands of: ";
1768 MI.dump();
1769 errs() << '\n';
1770 abort();
1771 }
1772 #endif
1773 }
1774
createX86MCCodeEmitter(const MCInstrInfo & MCII,MCContext & Ctx)1775 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1776 MCContext &Ctx) {
1777 return new X86MCCodeEmitter(MCII, Ctx);
1778 }
1779