xref: /llvm-project/bolt/lib/Target/X86/X86MCPlusBuilder.cpp (revision 89ca4eb002021b7e11037a5577e2ee9bc570be42)
1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file provides X86-specific MCPlus builder.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86InstrRelaxTables.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "X86MCSymbolizer.h"
17 #include "bolt/Core/MCPlus.h"
18 #include "bolt/Core/MCPlusBuilder.h"
19 #include "llvm/BinaryFormat/ELF.h"
20 #include "llvm/MC/MCContext.h"
21 #include "llvm/MC/MCFixupKindInfo.h"
22 #include "llvm/MC/MCInst.h"
23 #include "llvm/MC/MCInstBuilder.h"
24 #include "llvm/MC/MCInstrInfo.h"
25 #include "llvm/MC/MCRegister.h"
26 #include "llvm/MC/MCRegisterInfo.h"
27 #include "llvm/Support/CommandLine.h"
28 #include "llvm/Support/DataExtractor.h"
29 #include "llvm/Support/Debug.h"
30 #include "llvm/Support/Errc.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/ErrorOr.h"
33 #include <set>
34 
35 #define DEBUG_TYPE "mcplus"
36 
37 using namespace llvm;
38 using namespace bolt;
39 
40 namespace opts {
41 
42 extern cl::OptionCategory BoltOptCategory;
43 
44 static cl::opt<bool> X86StripRedundantAddressSize(
45     "x86-strip-redundant-address-size",
46     cl::desc("Remove redundant Address-Size override prefix"), cl::init(true),
47     cl::cat(BoltOptCategory));
48 
49 } // namespace opts
50 
51 namespace {
52 
53 unsigned getShortArithOpcode(unsigned Opcode) {
54   return X86::getShortOpcodeArith(Opcode);
55 }
56 
57 bool isMOVSX64rm32(const MCInst &Inst) {
58   return Inst.getOpcode() == X86::MOVSX64rm32;
59 }
60 
61 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; }
62 
63 bool isADDri(const MCInst &Inst) {
64   return Inst.getOpcode() == X86::ADD64ri32 ||
65          Inst.getOpcode() == X86::ADD64ri8;
66 }
67 
68 #define GET_INSTRINFO_OPERAND_TYPES_ENUM
69 #define GET_INSTRINFO_OPERAND_TYPE
70 #define GET_INSTRINFO_MEM_OPERAND_SIZE
71 #include "X86GenInstrInfo.inc"
72 
73 class X86MCPlusBuilder : public MCPlusBuilder {
74 public:
75   X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info,
76                    const MCRegisterInfo *RegInfo)
77       : MCPlusBuilder(Analysis, Info, RegInfo) {}
78 
79   std::unique_ptr<MCSymbolizer>
80   createTargetSymbolizer(BinaryFunction &Function) const override {
81     return std::make_unique<X86MCSymbolizer>(Function);
82   }
83 
84   bool isBranch(const MCInst &Inst) const override {
85     return Analysis->isBranch(Inst) && !isTailCall(Inst);
86   }
87 
88   bool isNoop(const MCInst &Inst) const override {
89     return X86::isNOP(Inst.getOpcode());
90   }
91 
92   unsigned getCondCode(const MCInst &Inst) const override {
93     unsigned Opcode = Inst.getOpcode();
94     if (X86::isJCC(Opcode))
95       return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm();
96     return X86::COND_INVALID;
97   }
98 
99   unsigned getInvertedCondCode(unsigned CC) const override {
100     switch (CC) {
101     default: return X86::COND_INVALID;
102     case X86::COND_E:  return X86::COND_NE;
103     case X86::COND_NE: return X86::COND_E;
104     case X86::COND_L:  return X86::COND_GE;
105     case X86::COND_LE: return X86::COND_G;
106     case X86::COND_G:  return X86::COND_LE;
107     case X86::COND_GE: return X86::COND_L;
108     case X86::COND_B:  return X86::COND_AE;
109     case X86::COND_BE: return X86::COND_A;
110     case X86::COND_A:  return X86::COND_BE;
111     case X86::COND_AE: return X86::COND_B;
112     case X86::COND_S:  return X86::COND_NS;
113     case X86::COND_NS: return X86::COND_S;
114     case X86::COND_P:  return X86::COND_NP;
115     case X86::COND_NP: return X86::COND_P;
116     case X86::COND_O:  return X86::COND_NO;
117     case X86::COND_NO: return X86::COND_O;
118     }
119   }
120 
121   unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
122     enum DecodedCondCode : uint8_t {
123       DCC_EQUAL = 0x1,
124       DCC_GREATER = 0x2,
125       DCC_LESSER = 0x4,
126       DCC_GREATER_OR_LESSER = 0x6,
127       DCC_UNSIGNED = 0x8,
128       DCC_SIGNED = 0x10,
129       DCC_INVALID = 0x20,
130     };
131 
132     auto decodeCondCode = [&](unsigned CC) -> uint8_t {
133       switch (CC) {
134       default: return DCC_INVALID;
135       case X86::COND_E: return DCC_EQUAL;
136       case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
137       case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
138       case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
139       case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
140       case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
141       case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
142       case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
143       case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
144       case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
145       }
146     };
147 
148     uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
149 
150     if (DCC & DCC_INVALID)
151       return X86::COND_INVALID;
152 
153     if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
154       return X86::COND_INVALID;
155 
156     switch (DCC) {
157     default: return X86::COND_INVALID;
158     case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
159     case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
160     case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
161     case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
162     case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
163     case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
164     case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
165     case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
166     case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
167     case DCC_EQUAL: return X86::COND_E;
168     case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
169     case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
170     case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
171     case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
172     }
173   }
174 
175   bool isValidCondCode(unsigned CC) const override {
176     return (CC != X86::COND_INVALID);
177   }
178 
179   bool isBreakpoint(const MCInst &Inst) const override {
180     return Inst.getOpcode() == X86::INT3;
181   }
182 
183   bool isPrefix(const MCInst &Inst) const override {
184     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
185     return X86II::isPrefix(Desc.TSFlags);
186   }
187 
188   bool isRep(const MCInst &Inst) const override {
189     return Inst.getFlags() == X86::IP_HAS_REPEAT;
190   }
191 
192   bool deleteREPPrefix(MCInst &Inst) const override {
193     if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
194       Inst.setFlags(0);
195       return true;
196     }
197     return false;
198   }
199 
200   // FIXME: For compatibility with old LLVM only!
201   bool isTerminator(const MCInst &Inst) const override {
202     unsigned Opcode = Inst.getOpcode();
203     return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) ||
204            X86::isUD2(Opcode);
205   }
206 
207   bool isIndirectCall(const MCInst &Inst) const override {
208     return isCall(Inst) &&
209            ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg());
210   }
211 
212   bool isPop(const MCInst &Inst) const override {
213     return getPopSize(Inst) == 0 ? false : true;
214   }
215 
216   bool isTerminateBranch(const MCInst &Inst) const override {
217     return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
218   }
219 
220   int getPopSize(const MCInst &Inst) const override {
221     switch (Inst.getOpcode()) {
222     case X86::POP16r:
223     case X86::POP16rmm:
224     case X86::POP16rmr:
225     case X86::POPF16:
226     case X86::POPA16:
227     case X86::POPDS16:
228     case X86::POPES16:
229     case X86::POPFS16:
230     case X86::POPGS16:
231     case X86::POPSS16:
232       return 2;
233     case X86::POP32r:
234     case X86::POP32rmm:
235     case X86::POP32rmr:
236     case X86::POPA32:
237     case X86::POPDS32:
238     case X86::POPES32:
239     case X86::POPF32:
240     case X86::POPFS32:
241     case X86::POPGS32:
242     case X86::POPSS32:
243       return 4;
244     case X86::POP64r:
245     case X86::POP64rmm:
246     case X86::POP64rmr:
247     case X86::POPF64:
248     case X86::POPFS64:
249     case X86::POPGS64:
250       return 8;
251     }
252     return 0;
253   }
254 
255   bool isPush(const MCInst &Inst) const override {
256     return getPushSize(Inst) == 0 ? false : true;
257   }
258 
259   int getPushSize(const MCInst &Inst) const override {
260     switch (Inst.getOpcode()) {
261     case X86::PUSH16i8:
262     case X86::PUSH16r:
263     case X86::PUSH16rmm:
264     case X86::PUSH16rmr:
265     case X86::PUSHA16:
266     case X86::PUSHCS16:
267     case X86::PUSHDS16:
268     case X86::PUSHES16:
269     case X86::PUSHF16:
270     case X86::PUSHFS16:
271     case X86::PUSHGS16:
272     case X86::PUSHSS16:
273     case X86::PUSH16i:
274       return 2;
275     case X86::PUSH32i8:
276     case X86::PUSH32r:
277     case X86::PUSH32rmm:
278     case X86::PUSH32rmr:
279     case X86::PUSHA32:
280     case X86::PUSHCS32:
281     case X86::PUSHDS32:
282     case X86::PUSHES32:
283     case X86::PUSHF32:
284     case X86::PUSHFS32:
285     case X86::PUSHGS32:
286     case X86::PUSHSS32:
287     case X86::PUSH32i:
288       return 4;
289     case X86::PUSH64i32:
290     case X86::PUSH64i8:
291     case X86::PUSH64r:
292     case X86::PUSH64rmm:
293     case X86::PUSH64rmr:
294     case X86::PUSHF64:
295     case X86::PUSHFS64:
296     case X86::PUSHGS64:
297       return 8;
298     }
299     return 0;
300   }
301 
302   bool isSUB(const MCInst &Inst) const override {
303     return X86::isSUB(Inst.getOpcode());
304   }
305 
306   bool isLEA64r(const MCInst &Inst) const override {
307     return Inst.getOpcode() == X86::LEA64r;
308   }
309 
310   bool isLeave(const MCInst &Inst) const override {
311     return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
312   }
313 
314   bool isMoveMem2Reg(const MCInst &Inst) const override {
315     switch (Inst.getOpcode()) {
316     case X86::MOV16rm:
317     case X86::MOV32rm:
318     case X86::MOV64rm:
319       return true;
320     }
321     return false;
322   }
323 
324   bool isUnsupportedBranch(unsigned Opcode) const override {
325     switch (Opcode) {
326     default:
327       return false;
328     case X86::LOOP:
329     case X86::LOOPE:
330     case X86::LOOPNE:
331     case X86::JECXZ:
332     case X86::JRCXZ:
333       return true;
334     }
335   }
336 
337   bool isLoad(const MCInst &Inst) const override {
338     if (isPop(Inst))
339       return true;
340 
341     int MemOpNo = getMemoryOperandNo(Inst);
342     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
343 
344     if (MemOpNo == -1)
345       return false;
346 
347     return MCII.mayLoad();
348   }
349 
350   bool isStore(const MCInst &Inst) const override {
351     if (isPush(Inst))
352       return true;
353 
354     int MemOpNo = getMemoryOperandNo(Inst);
355     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
356 
357     if (MemOpNo == -1)
358       return false;
359 
360     return MCII.mayStore();
361   }
362 
363   bool isCleanRegXOR(const MCInst &Inst) const override {
364     switch (Inst.getOpcode()) {
365     case X86::XOR16rr:
366     case X86::XOR32rr:
367     case X86::XOR64rr:
368       break;
369     default:
370       return false;
371     }
372     return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg());
373   }
374 
375   bool isPacked(const MCInst &Inst) const override {
376     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
377     return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
378   }
379 
380   bool shouldRecordCodeRelocation(uint64_t RelType) const override {
381     switch (RelType) {
382     case ELF::R_X86_64_8:
383     case ELF::R_X86_64_16:
384     case ELF::R_X86_64_32:
385     case ELF::R_X86_64_32S:
386     case ELF::R_X86_64_64:
387     case ELF::R_X86_64_PC8:
388     case ELF::R_X86_64_PC32:
389     case ELF::R_X86_64_PC64:
390     case ELF::R_X86_64_GOTPCRELX:
391     case ELF::R_X86_64_REX_GOTPCRELX:
392       return true;
393     case ELF::R_X86_64_PLT32:
394     case ELF::R_X86_64_GOTPCREL:
395     case ELF::R_X86_64_TPOFF32:
396     case ELF::R_X86_64_GOTTPOFF:
397       return false;
398     default:
399       llvm_unreachable("Unexpected x86 relocation type in code");
400     }
401   }
402 
403   unsigned getTrapFillValue() const override { return 0xCC; }
404 
405   struct IndJmpMatcherFrag1 : MCInstMatcher {
406     std::unique_ptr<MCInstMatcher> Base;
407     std::unique_ptr<MCInstMatcher> Scale;
408     std::unique_ptr<MCInstMatcher> Index;
409     std::unique_ptr<MCInstMatcher> Offset;
410 
411     IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
412                        std::unique_ptr<MCInstMatcher> Scale,
413                        std::unique_ptr<MCInstMatcher> Index,
414                        std::unique_ptr<MCInstMatcher> Offset)
415         : Base(std::move(Base)), Scale(std::move(Scale)),
416           Index(std::move(Index)), Offset(std::move(Offset)) {}
417 
418     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
419                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
420       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
421         return false;
422 
423       if (CurInst->getOpcode() != X86::JMP64m)
424         return false;
425 
426       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
427       if (MemOpNo == -1)
428         return false;
429 
430       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
431         return false;
432       if (!Scale->match(MRI, MIB, this->InstrWindow,
433                         MemOpNo + X86::AddrScaleAmt))
434         return false;
435       if (!Index->match(MRI, MIB, this->InstrWindow,
436                         MemOpNo + X86::AddrIndexReg))
437         return false;
438       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
439         return false;
440       return true;
441     }
442 
443     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
444       MIB.addAnnotation(*CurInst, Annotation, true);
445       Base->annotate(MIB, Annotation);
446       Scale->annotate(MIB, Annotation);
447       Index->annotate(MIB, Annotation);
448       Offset->annotate(MIB, Annotation);
449     }
450   };
451 
452   std::unique_ptr<MCInstMatcher>
453   matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
454               std::unique_ptr<MCInstMatcher> Scale,
455               std::unique_ptr<MCInstMatcher> Index,
456               std::unique_ptr<MCInstMatcher> Offset) const override {
457     return std::unique_ptr<MCInstMatcher>(
458         new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
459                                std::move(Index), std::move(Offset)));
460   }
461 
462   struct IndJmpMatcherFrag2 : MCInstMatcher {
463     std::unique_ptr<MCInstMatcher> Reg;
464 
465     IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
466         : Reg(std::move(Reg)) {}
467 
468     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
469                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
470       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
471         return false;
472 
473       if (CurInst->getOpcode() != X86::JMP64r)
474         return false;
475 
476       return Reg->match(MRI, MIB, this->InstrWindow, 0);
477     }
478 
479     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
480       MIB.addAnnotation(*CurInst, Annotation, true);
481       Reg->annotate(MIB, Annotation);
482     }
483   };
484 
485   std::unique_ptr<MCInstMatcher>
486   matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
487     return std::unique_ptr<MCInstMatcher>(
488         new IndJmpMatcherFrag2(std::move(Target)));
489   }
490 
491   struct LoadMatcherFrag1 : MCInstMatcher {
492     std::unique_ptr<MCInstMatcher> Base;
493     std::unique_ptr<MCInstMatcher> Scale;
494     std::unique_ptr<MCInstMatcher> Index;
495     std::unique_ptr<MCInstMatcher> Offset;
496 
497     LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
498                      std::unique_ptr<MCInstMatcher> Scale,
499                      std::unique_ptr<MCInstMatcher> Index,
500                      std::unique_ptr<MCInstMatcher> Offset)
501         : Base(std::move(Base)), Scale(std::move(Scale)),
502           Index(std::move(Index)), Offset(std::move(Offset)) {}
503 
504     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
505                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
506       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
507         return false;
508 
509       if (CurInst->getOpcode() != X86::MOV64rm &&
510           CurInst->getOpcode() != X86::MOVSX64rm32)
511         return false;
512 
513       int MemOpNo = MIB.getMemoryOperandNo(*CurInst);
514       if (MemOpNo == -1)
515         return false;
516 
517       if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
518         return false;
519       if (!Scale->match(MRI, MIB, this->InstrWindow,
520                         MemOpNo + X86::AddrScaleAmt))
521         return false;
522       if (!Index->match(MRI, MIB, this->InstrWindow,
523                         MemOpNo + X86::AddrIndexReg))
524         return false;
525       if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
526         return false;
527       return true;
528     }
529 
530     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
531       MIB.addAnnotation(*CurInst, Annotation, true);
532       Base->annotate(MIB, Annotation);
533       Scale->annotate(MIB, Annotation);
534       Index->annotate(MIB, Annotation);
535       Offset->annotate(MIB, Annotation);
536     }
537   };
538 
539   std::unique_ptr<MCInstMatcher>
540   matchLoad(std::unique_ptr<MCInstMatcher> Base,
541             std::unique_ptr<MCInstMatcher> Scale,
542             std::unique_ptr<MCInstMatcher> Index,
543             std::unique_ptr<MCInstMatcher> Offset) const override {
544     return std::unique_ptr<MCInstMatcher>(
545         new LoadMatcherFrag1(std::move(Base), std::move(Scale),
546                              std::move(Index), std::move(Offset)));
547   }
548 
549   struct AddMatcher : MCInstMatcher {
550     std::unique_ptr<MCInstMatcher> A;
551     std::unique_ptr<MCInstMatcher> B;
552 
553     AddMatcher(std::unique_ptr<MCInstMatcher> A,
554                std::unique_ptr<MCInstMatcher> B)
555         : A(std::move(A)), B(std::move(B)) {}
556 
557     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
558                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
559       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
560         return false;
561 
562       if (CurInst->getOpcode() == X86::ADD64rr ||
563           CurInst->getOpcode() == X86::ADD64rr_DB ||
564           CurInst->getOpcode() == X86::ADD64rr_REV) {
565         if (!A->match(MRI, MIB, this->InstrWindow, 1)) {
566           if (!B->match(MRI, MIB, this->InstrWindow, 1))
567             return false;
568           return A->match(MRI, MIB, this->InstrWindow, 2);
569         }
570 
571         if (B->match(MRI, MIB, this->InstrWindow, 2))
572           return true;
573 
574         if (!B->match(MRI, MIB, this->InstrWindow, 1))
575           return false;
576         return A->match(MRI, MIB, this->InstrWindow, 2);
577       }
578 
579       return false;
580     }
581 
582     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
583       MIB.addAnnotation(*CurInst, Annotation, true);
584       A->annotate(MIB, Annotation);
585       B->annotate(MIB, Annotation);
586     }
587   };
588 
589   std::unique_ptr<MCInstMatcher>
590   matchAdd(std::unique_ptr<MCInstMatcher> A,
591            std::unique_ptr<MCInstMatcher> B) const override {
592     return std::unique_ptr<MCInstMatcher>(
593         new AddMatcher(std::move(A), std::move(B)));
594   }
595 
596   struct LEAMatcher : MCInstMatcher {
597     std::unique_ptr<MCInstMatcher> Target;
598 
599     LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
600         : Target(std::move(Target)) {}
601 
602     bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
603                MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
604       if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
605         return false;
606 
607       if (CurInst->getOpcode() != X86::LEA64r)
608         return false;
609 
610       if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 ||
611           CurInst->getOperand(1 + X86::AddrIndexReg).getReg() !=
612               X86::NoRegister ||
613           (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() !=
614                X86::NoRegister &&
615            CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP))
616         return false;
617 
618       return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp);
619     }
620 
621     void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
622       MIB.addAnnotation(*CurInst, Annotation, true);
623       Target->annotate(MIB, Annotation);
624     }
625   };
626 
627   std::unique_ptr<MCInstMatcher>
628   matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
629     return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
630   }
631 
632   bool hasPCRelOperand(const MCInst &Inst) const override {
633     for (const MCOperand &Operand : Inst)
634       if (Operand.isReg() && Operand.getReg() == X86::RIP)
635         return true;
636     return false;
637   }
638 
639   int getMemoryOperandNo(const MCInst &Inst) const override {
640     unsigned Opcode = Inst.getOpcode();
641     const MCInstrDesc &Desc = Info->get(Opcode);
642     int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags);
643     if (MemOpNo >= 0)
644       MemOpNo += X86II::getOperandBias(Desc);
645     return MemOpNo;
646   }
647 
648   bool hasEVEXEncoding(const MCInst &Inst) const override {
649     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
650     return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
651   }
652 
653   bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
654     const auto *I = Insts.begin();
655     while (I != Insts.end() && isPrefix(*I))
656       ++I;
657     if (I == Insts.end())
658       return false;
659 
660     const MCInst &FirstInst = *I;
661     ++I;
662     while (I != Insts.end() && isPrefix(*I))
663       ++I;
664     if (I == Insts.end())
665       return false;
666     const MCInst &SecondInst = *I;
667 
668     if (!isConditionalBranch(SecondInst))
669       return false;
670     // Cannot fuse if the first instruction uses RIP-relative memory.
671     if (hasPCRelOperand(FirstInst))
672       return false;
673 
674     const X86::FirstMacroFusionInstKind CmpKind =
675         X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode());
676     if (CmpKind == X86::FirstMacroFusionInstKind::Invalid)
677       return false;
678 
679     X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst));
680     X86::SecondMacroFusionInstKind BranchKind =
681         X86::classifySecondCondCodeInMacroFusion(CC);
682     if (BranchKind == X86::SecondMacroFusionInstKind::Invalid)
683       return false;
684     return X86::isMacroFused(CmpKind, BranchKind);
685   }
686 
687   std::optional<X86MemOperand>
688   evaluateX86MemoryOperand(const MCInst &Inst) const override {
689     int MemOpNo = getMemoryOperandNo(Inst);
690     if (MemOpNo < 0)
691       return std::nullopt;
692     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
693 
694     if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
695       return std::nullopt;
696 
697     const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg);
698     const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt);
699     const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg);
700     const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp);
701     const MCOperand &Segment =
702         Inst.getOperand(MemOpOffset + X86::AddrSegmentReg);
703 
704     // Make sure it is a well-formed memory operand.
705     if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
706         (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
707       return std::nullopt;
708 
709     X86MemOperand MO;
710     MO.BaseRegNum = Base.getReg();
711     MO.ScaleImm = Scale.getImm();
712     MO.IndexRegNum = Index.getReg();
713     MO.DispImm = Disp.isImm() ? Disp.getImm() : 0;
714     MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr;
715     MO.SegRegNum = Segment.getReg();
716     return MO;
717   }
718 
719   bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
720                                 uint64_t Address,
721                                 uint64_t Size) const override {
722     std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
723     if (!MO)
724       return false;
725 
726     // Make sure it's a well-formed addressing we can statically evaluate.
727     if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) ||
728         MO->IndexRegNum != X86::NoRegister ||
729         MO->SegRegNum != X86::NoRegister || MO->DispExpr)
730       return false;
731 
732     Target = MO->DispImm;
733     if (MO->BaseRegNum == X86::RIP) {
734       assert(Size != 0 && "instruction size required in order to statically "
735                           "evaluate RIP-relative address");
736       Target += Address + Size;
737     }
738     return true;
739   }
740 
741   MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
742     int MemOpNo = getMemoryOperandNo(Inst);
743     if (MemOpNo < 0)
744       return Inst.end();
745     return Inst.begin() + (MemOpNo + X86::AddrDisp);
746   }
747 
748   bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
749     MCOperand *OI = getMemOperandDisp(Inst);
750     if (OI == Inst.end())
751       return false;
752     *OI = Operand;
753     return true;
754   }
755 
756   /// Get the registers used as function parameters.
757   /// This function is specific to the x86_64 abi on Linux.
758   BitVector getRegsUsedAsParams() const override {
759     BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
760     Regs |= getAliases(X86::RSI);
761     Regs |= getAliases(X86::RDI);
762     Regs |= getAliases(X86::RDX);
763     Regs |= getAliases(X86::RCX);
764     Regs |= getAliases(X86::R8);
765     Regs |= getAliases(X86::R9);
766     return Regs;
767   }
768 
769   void getCalleeSavedRegs(BitVector &Regs) const override {
770     Regs |= getAliases(X86::RBX);
771     Regs |= getAliases(X86::RBP);
772     Regs |= getAliases(X86::R12);
773     Regs |= getAliases(X86::R13);
774     Regs |= getAliases(X86::R14);
775     Regs |= getAliases(X86::R15);
776   }
777 
778   void getDefaultDefIn(BitVector &Regs) const override {
779     assert(Regs.size() >= RegInfo->getNumRegs() &&
780            "The size of BitVector is less than RegInfo->getNumRegs().");
781     Regs.set(X86::RAX);
782     Regs.set(X86::RCX);
783     Regs.set(X86::RDX);
784     Regs.set(X86::RSI);
785     Regs.set(X86::RDI);
786     Regs.set(X86::R8);
787     Regs.set(X86::R9);
788     Regs.set(X86::XMM0);
789     Regs.set(X86::XMM1);
790     Regs.set(X86::XMM2);
791     Regs.set(X86::XMM3);
792     Regs.set(X86::XMM4);
793     Regs.set(X86::XMM5);
794     Regs.set(X86::XMM6);
795     Regs.set(X86::XMM7);
796   }
797 
798   void getDefaultLiveOut(BitVector &Regs) const override {
799     assert(Regs.size() >= RegInfo->getNumRegs() &&
800            "The size of BitVector is less than RegInfo->getNumRegs().");
801     Regs |= getAliases(X86::RAX);
802     Regs |= getAliases(X86::RDX);
803     Regs |= getAliases(X86::RCX);
804     Regs |= getAliases(X86::XMM0);
805     Regs |= getAliases(X86::XMM1);
806   }
807 
808   void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
809     if (IncludeAlias) {
810       Regs |= getAliases(X86::RAX);
811       Regs |= getAliases(X86::RBX);
812       Regs |= getAliases(X86::RBP);
813       Regs |= getAliases(X86::RSI);
814       Regs |= getAliases(X86::RDI);
815       Regs |= getAliases(X86::RDX);
816       Regs |= getAliases(X86::RCX);
817       Regs |= getAliases(X86::R8);
818       Regs |= getAliases(X86::R9);
819       Regs |= getAliases(X86::R10);
820       Regs |= getAliases(X86::R11);
821       Regs |= getAliases(X86::R12);
822       Regs |= getAliases(X86::R13);
823       Regs |= getAliases(X86::R14);
824       Regs |= getAliases(X86::R15);
825       return;
826     }
827     Regs.set(X86::RAX);
828     Regs.set(X86::RBX);
829     Regs.set(X86::RBP);
830     Regs.set(X86::RSI);
831     Regs.set(X86::RDI);
832     Regs.set(X86::RDX);
833     Regs.set(X86::RCX);
834     Regs.set(X86::R8);
835     Regs.set(X86::R9);
836     Regs.set(X86::R10);
837     Regs.set(X86::R11);
838     Regs.set(X86::R12);
839     Regs.set(X86::R13);
840     Regs.set(X86::R14);
841     Regs.set(X86::R15);
842   }
843 
844   void getClassicGPRegs(BitVector &Regs) const override {
845     Regs |= getAliases(X86::RAX);
846     Regs |= getAliases(X86::RBX);
847     Regs |= getAliases(X86::RBP);
848     Regs |= getAliases(X86::RSI);
849     Regs |= getAliases(X86::RDI);
850     Regs |= getAliases(X86::RDX);
851     Regs |= getAliases(X86::RCX);
852   }
853 
854   void getRepRegs(BitVector &Regs) const override {
855     Regs |= getAliases(X86::RCX);
856   }
857 
858   MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
859     Reg = getX86SubSuperRegister(Reg, Size * 8);
860     assert((Reg != X86::NoRegister) && "Invalid register");
861     return Reg;
862   }
863 
864   bool isUpper8BitReg(MCPhysReg Reg) const override {
865     switch (Reg) {
866     case X86::AH:
867     case X86::BH:
868     case X86::CH:
869     case X86::DH:
870       return true;
871     default:
872       return false;
873     }
874   }
875 
876   bool cannotUseREX(const MCInst &Inst) const override {
877     switch (Inst.getOpcode()) {
878     case X86::MOV8mr_NOREX:
879     case X86::MOV8rm_NOREX:
880     case X86::MOV8rr_NOREX:
881     case X86::MOVSX32rm8_NOREX:
882     case X86::MOVSX32rr8_NOREX:
883     case X86::MOVZX32rm8_NOREX:
884     case X86::MOVZX32rr8_NOREX:
885     case X86::MOV8mr:
886     case X86::MOV8rm:
887     case X86::MOV8rr:
888     case X86::MOVSX32rm8:
889     case X86::MOVSX32rr8:
890     case X86::MOVZX32rm8:
891     case X86::MOVZX32rr8:
892     case X86::TEST8ri:
893       for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) {
894         if (!Operand.isReg())
895           continue;
896         if (isUpper8BitReg(Operand.getReg()))
897           return true;
898       }
899       [[fallthrough]];
900     default:
901       return false;
902     }
903   }
904 
905   static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) {
906     using namespace llvm::X86;
907     int OpType = getOperandType(Inst.getOpcode(), MemOpNo);
908     return getMemOperandSize(OpType) / 8;
909   }
910 
911   /// Classifying a stack access as *not* "SIMPLE" here means we don't know how
912   /// to change this instruction memory access. It will disable any changes to
913   /// the stack layout, so we can't do the most aggressive form of shrink
914   /// wrapping. We must do so in a way that keeps the original stack layout.
915   /// Otherwise you need to adjust the offset of all instructions accessing the
916   /// stack: we can't do that anymore because there is one instruction that is
917   /// not simple. There are other implications as well. We have heuristics to
918   /// detect when a register is callee-saved and thus eligible for shrink
919   /// wrapping. If you are restoring a register using a non-simple stack access,
920   /// then it is classified as NOT callee-saved, and it disables shrink wrapping
921   /// for *that* register (but not for others).
922   ///
923   /// Classifying a stack access as "size 0" or detecting an indexed memory
924   /// access (to address a vector, for example) here means we know there is a
925   /// stack access, but we can't quite understand how wide is the access in
926   /// bytes. This is very serious because we can't understand how memory
927   /// accesses alias with each other for this function. This will essentially
928   /// disable not only shrink wrapping but all frame analysis, it will fail it
929   /// as "we don't understand this function and we give up on it".
930   bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
931                      bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
932                      uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
933                      bool &IsSimple, bool &IsIndexed) const override {
934     // Detect simple push/pop cases first
935     if (int Sz = getPushSize(Inst)) {
936       IsLoad = false;
937       IsStore = true;
938       IsStoreFromReg = true;
939       StackPtrReg = X86::RSP;
940       StackOffset = -Sz;
941       Size = Sz;
942       IsSimple = true;
943       if (Inst.getOperand(0).isImm())
944         SrcImm = Inst.getOperand(0).getImm();
945       else if (Inst.getOperand(0).isReg())
946         Reg = Inst.getOperand(0).getReg();
947       else
948         IsSimple = false;
949 
950       return true;
951     }
952     if (int Sz = getPopSize(Inst)) {
953       IsLoad = true;
954       IsStore = false;
955       if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) {
956         IsSimple = false;
957       } else {
958         Reg = Inst.getOperand(0).getReg();
959         IsSimple = true;
960       }
961       StackPtrReg = X86::RSP;
962       StackOffset = 0;
963       Size = Sz;
964       return true;
965     }
966 
967     struct InstInfo {
968       // Size in bytes that Inst loads from memory.
969       uint8_t DataSize;
970       bool IsLoad;
971       bool IsStore;
972       bool StoreFromReg;
973       bool Simple;
974     };
975 
976     InstInfo I;
977     int MemOpNo = getMemoryOperandNo(Inst);
978     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
979     // If it is not dealing with a memory operand, we discard it
980     if (MemOpNo == -1 || MCII.isCall())
981       return false;
982 
983     switch (Inst.getOpcode()) {
984     default: {
985       bool IsLoad = MCII.mayLoad();
986       bool IsStore = MCII.mayStore();
987       // Is it LEA? (deals with memory but is not loading nor storing)
988       if (!IsLoad && !IsStore) {
989         I = {0, IsLoad, IsStore, false, false};
990         break;
991       }
992       uint8_t Sz = getMemDataSize(Inst, MemOpNo);
993       I = {Sz, IsLoad, IsStore, false, false};
994       break;
995     }
996     // Report simple stack accesses
997     case X86::MOV8rm: I = {1, true, false, false, true}; break;
998     case X86::MOV16rm: I = {2, true, false, false, true}; break;
999     case X86::MOV32rm: I = {4, true, false, false, true}; break;
1000     case X86::MOV64rm: I = {8, true, false, false, true}; break;
1001     case X86::MOV8mr: I = {1, false, true, true, true};  break;
1002     case X86::MOV16mr: I = {2, false, true, true, true};  break;
1003     case X86::MOV32mr: I = {4, false, true, true, true};  break;
1004     case X86::MOV64mr: I = {8, false, true, true, true};  break;
1005     case X86::MOV8mi: I = {1, false, true, false, true}; break;
1006     case X86::MOV16mi: I = {2, false, true, false, true}; break;
1007     case X86::MOV32mi: I = {4, false, true, false, true}; break;
1008     } // end switch (Inst.getOpcode())
1009 
1010     std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1011     if (!MO) {
1012       LLVM_DEBUG(dbgs() << "Evaluate failed on ");
1013       LLVM_DEBUG(Inst.dump());
1014       return false;
1015     }
1016 
1017     // Make sure it's a stack access
1018     if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP)
1019       return false;
1020 
1021     IsLoad = I.IsLoad;
1022     IsStore = I.IsStore;
1023     IsStoreFromReg = I.StoreFromReg;
1024     Size = I.DataSize;
1025     IsSimple = I.Simple;
1026     StackPtrReg = MO->BaseRegNum;
1027     StackOffset = MO->DispImm;
1028     IsIndexed =
1029         MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister;
1030 
1031     if (!I.Simple)
1032       return true;
1033 
1034     // Retrieve related register in simple MOV from/to stack operations.
1035     unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1036     if (I.IsLoad) {
1037       MCOperand RegOpnd = Inst.getOperand(0);
1038       assert(RegOpnd.isReg() && "unexpected destination operand");
1039       Reg = RegOpnd.getReg();
1040     } else if (I.IsStore) {
1041       MCOperand SrcOpnd =
1042           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1043       if (I.StoreFromReg) {
1044         assert(SrcOpnd.isReg() && "unexpected source operand");
1045         Reg = SrcOpnd.getReg();
1046       } else {
1047         assert(SrcOpnd.isImm() && "unexpected source operand");
1048         SrcImm = SrcOpnd.getImm();
1049       }
1050     }
1051 
1052     return true;
1053   }
1054 
1055   void changeToPushOrPop(MCInst &Inst) const override {
1056     assert(!isPush(Inst) && !isPop(Inst));
1057 
1058     struct InstInfo {
1059       // Size in bytes that Inst loads from memory.
1060       uint8_t DataSize;
1061       bool IsLoad;
1062       bool StoreFromReg;
1063     };
1064 
1065     InstInfo I;
1066     switch (Inst.getOpcode()) {
1067     default: {
1068       llvm_unreachable("Unhandled opcode");
1069       return;
1070     }
1071     case X86::MOV16rm: I = {2, true, false}; break;
1072     case X86::MOV32rm: I = {4, true, false}; break;
1073     case X86::MOV64rm: I = {8, true, false}; break;
1074     case X86::MOV16mr: I = {2, false, true};  break;
1075     case X86::MOV32mr: I = {4, false, true};  break;
1076     case X86::MOV64mr: I = {8, false, true};  break;
1077     case X86::MOV16mi: I = {2, false, false}; break;
1078     case X86::MOV32mi: I = {4, false, false}; break;
1079     } // end switch (Inst.getOpcode())
1080 
1081     std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1082     if (!MO) {
1083       llvm_unreachable("Evaluate failed");
1084       return;
1085     }
1086     // Make sure it's a stack access
1087     if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) {
1088       llvm_unreachable("Not a stack access");
1089       return;
1090     }
1091 
1092     unsigned MemOpOffset = getMemoryOperandNo(Inst);
1093     unsigned NewOpcode = 0;
1094     if (I.IsLoad) {
1095       switch (I.DataSize) {
1096       case 2: NewOpcode = X86::POP16r; break;
1097       case 4: NewOpcode = X86::POP32r; break;
1098       case 8: NewOpcode = X86::POP64r; break;
1099       default:
1100         llvm_unreachable("Unexpected size");
1101       }
1102       unsigned RegOpndNum = Inst.getOperand(0).getReg();
1103       Inst.clear();
1104       Inst.setOpcode(NewOpcode);
1105       Inst.addOperand(MCOperand::createReg(RegOpndNum));
1106     } else {
1107       MCOperand SrcOpnd =
1108           Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1);
1109       if (I.StoreFromReg) {
1110         switch (I.DataSize) {
1111         case 2: NewOpcode = X86::PUSH16r; break;
1112         case 4: NewOpcode = X86::PUSH32r; break;
1113         case 8: NewOpcode = X86::PUSH64r; break;
1114         default:
1115           llvm_unreachable("Unexpected size");
1116         }
1117         assert(SrcOpnd.isReg() && "Unexpected source operand");
1118         unsigned RegOpndNum = SrcOpnd.getReg();
1119         Inst.clear();
1120         Inst.setOpcode(NewOpcode);
1121         Inst.addOperand(MCOperand::createReg(RegOpndNum));
1122       } else {
1123         switch (I.DataSize) {
1124         case 2: NewOpcode = X86::PUSH16i8; break;
1125         case 4: NewOpcode = X86::PUSH32i8; break;
1126         case 8: NewOpcode = X86::PUSH64i32; break;
1127         default:
1128           llvm_unreachable("Unexpected size");
1129         }
1130         assert(SrcOpnd.isImm() && "Unexpected source operand");
1131         int64_t SrcImm = SrcOpnd.getImm();
1132         Inst.clear();
1133         Inst.setOpcode(NewOpcode);
1134         Inst.addOperand(MCOperand::createImm(SrcImm));
1135       }
1136     }
1137   }
1138 
1139   bool isStackAdjustment(const MCInst &Inst) const override {
1140     switch (Inst.getOpcode()) {
1141     default:
1142       return false;
1143     case X86::SUB64ri32:
1144     case X86::SUB64ri8:
1145     case X86::ADD64ri32:
1146     case X86::ADD64ri8:
1147     case X86::LEA64r:
1148       break;
1149     }
1150 
1151     return any_of(defOperands(Inst), [](const MCOperand &Op) {
1152       return Op.isReg() && Op.getReg() == X86::RSP;
1153     });
1154   }
1155 
1156   bool
1157   evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output,
1158                           std::pair<MCPhysReg, int64_t> Input1,
1159                           std::pair<MCPhysReg, int64_t> Input2) const override {
1160 
1161     auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1162       if (Reg == Input1.first)
1163         return Input1.second;
1164       if (Reg == Input2.first)
1165         return Input2.second;
1166       return make_error_code(errc::result_out_of_range);
1167     };
1168 
1169     switch (Inst.getOpcode()) {
1170     default:
1171       return false;
1172 
1173     case X86::SUB64ri32:
1174     case X86::SUB64ri8:
1175       if (!Inst.getOperand(2).isImm())
1176         return false;
1177       if (ErrorOr<int64_t> InputVal =
1178               getOperandVal(Inst.getOperand(1).getReg()))
1179         Output = *InputVal - Inst.getOperand(2).getImm();
1180       else
1181         return false;
1182       break;
1183     case X86::ADD64ri32:
1184     case X86::ADD64ri8:
1185       if (!Inst.getOperand(2).isImm())
1186         return false;
1187       if (ErrorOr<int64_t> InputVal =
1188               getOperandVal(Inst.getOperand(1).getReg()))
1189         Output = *InputVal + Inst.getOperand(2).getImm();
1190       else
1191         return false;
1192       break;
1193     case X86::ADD64i32:
1194       if (!Inst.getOperand(0).isImm())
1195         return false;
1196       if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1197         Output = *InputVal + Inst.getOperand(0).getImm();
1198       else
1199         return false;
1200       break;
1201 
1202     case X86::LEA64r: {
1203       std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1204       if (!MO)
1205         return false;
1206 
1207       if (MO->BaseRegNum == X86::NoRegister ||
1208           MO->IndexRegNum != X86::NoRegister ||
1209           MO->SegRegNum != X86::NoRegister || MO->DispExpr)
1210         return false;
1211 
1212       if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum))
1213         Output = *InputVal + MO->DispImm;
1214       else
1215         return false;
1216 
1217       break;
1218     }
1219     }
1220     return true;
1221   }
1222 
1223   bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1224                       MCPhysReg &To) const override {
1225     switch (Inst.getOpcode()) {
1226     default:
1227       return false;
1228     case X86::LEAVE:
1229     case X86::LEAVE64:
1230       To = getStackPointer();
1231       From = getFramePointer();
1232       return true;
1233     case X86::MOV64rr:
1234       To = Inst.getOperand(0).getReg();
1235       From = Inst.getOperand(1).getReg();
1236       return true;
1237     }
1238   }
1239 
1240   MCPhysReg getStackPointer() const override { return X86::RSP; }
1241   MCPhysReg getFramePointer() const override { return X86::RBP; }
1242   MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1243 
1244   bool escapesVariable(const MCInst &Inst,
1245                        bool HasFramePointer) const override {
1246     int MemOpNo = getMemoryOperandNo(Inst);
1247     const MCInstrDesc &MCII = Info->get(Inst.getOpcode());
1248     const unsigned NumDefs = MCII.getNumDefs();
1249     static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
1250                                  getAliases(X86::RBP));
1251     static BitVector SPAliases(getAliases(X86::RSP));
1252 
1253     // FIXME: PUSH can be technically a leak, but let's ignore this for now
1254     // because a lot of harmless prologue code will spill SP to the stack.
1255     // Unless push is clearly pushing an object address to the stack as
1256     // demonstrated by having a MemOp.
1257     bool IsPush = isPush(Inst);
1258     if (IsPush && MemOpNo == -1)
1259       return false;
1260 
1261     // We use this to detect LEA (has memop but does not access mem)
1262     bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1263     bool DoesLeak = false;
1264     for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1265       // Ignore if SP/BP is used to dereference memory -- that's fine
1266       if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1267           I <= MemOpNo + 5)
1268         continue;
1269       // Ignore if someone is writing to SP/BP
1270       if (I < static_cast<int>(NumDefs))
1271         continue;
1272 
1273       const MCOperand &Operand = Inst.getOperand(I);
1274       if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1275         DoesLeak = true;
1276         break;
1277       }
1278       if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1279         DoesLeak = true;
1280         break;
1281       }
1282     }
1283 
1284     // If potential leak, check if it is not just writing to itself/sp/bp
1285     if (DoesLeak) {
1286       DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) {
1287         assert(Operand.isReg());
1288         MCPhysReg Reg = Operand.getReg();
1289         return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg];
1290       });
1291     }
1292     return DoesLeak;
1293   }
1294 
1295   bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1296     unsigned ImmOpNo = -1U;
1297     int MemOpNo = getMemoryOperandNo(Inst);
1298     if (MemOpNo != -1)
1299       ImmOpNo = MemOpNo + X86::AddrDisp;
1300     else
1301       for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1302            ++Index)
1303         if (Inst.getOperand(Index).isImm())
1304           ImmOpNo = Index;
1305     if (ImmOpNo == -1U)
1306       return false;
1307 
1308     MCOperand &Operand = Inst.getOperand(ImmOpNo);
1309     Amt += Operand.getImm();
1310     Operand.setImm(Amt);
1311     // Check for the need for relaxation
1312     if (int64_t(Amt) == int64_t(int8_t(Amt)))
1313       return true;
1314 
1315     // Relax instruction
1316     switch (Inst.getOpcode()) {
1317     case X86::SUB64ri8:
1318       Inst.setOpcode(X86::SUB64ri32);
1319       break;
1320     case X86::ADD64ri8:
1321       Inst.setOpcode(X86::ADD64ri32);
1322       break;
1323     default:
1324       // No need for relaxation
1325       break;
1326     }
1327     return true;
1328   }
1329 
1330   /// TODO: this implementation currently works for the most common opcodes that
1331   /// load from memory. It can be extended to work with memory store opcodes as
1332   /// well as more memory load opcodes.
1333   bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1334                                 uint64_t Offset) const override {
1335     enum CheckSignExt : uint8_t {
1336       NOCHECK = 0,
1337       CHECK8,
1338       CHECK32,
1339     };
1340 
1341     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1342     struct InstInfo {
1343       // Size in bytes that Inst loads from memory.
1344       uint8_t DataSize;
1345 
1346       // True when the target operand has to be duplicated because the opcode
1347       // expects a LHS operand.
1348       bool HasLHS;
1349 
1350       // List of checks and corresponding opcodes to be used. We try to use the
1351       // smallest possible immediate value when various sizes are available,
1352       // hence we may need to check whether a larger constant fits in a smaller
1353       // immediate.
1354       CheckList Checks;
1355     };
1356 
1357     InstInfo I;
1358 
1359     switch (Inst.getOpcode()) {
1360     default: {
1361       switch (getPopSize(Inst)) {
1362       case 2:            I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1363       case 4:            I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1364       case 8:            I = {8, false, {{CHECK32, X86::MOV64ri32},
1365                                          {NOCHECK, X86::MOV64rm}}};  break;
1366       default:           return false;
1367       }
1368       break;
1369     }
1370 
1371     // MOV
1372     case X86::MOV8rm:      I = {1, false, {{NOCHECK, X86::MOV8ri}}};   break;
1373     case X86::MOV16rm:     I = {2, false, {{NOCHECK, X86::MOV16ri}}};  break;
1374     case X86::MOV32rm:     I = {4, false, {{NOCHECK, X86::MOV32ri}}};  break;
1375     case X86::MOV64rm:     I = {8, false, {{CHECK32, X86::MOV64ri32},
1376                                            {NOCHECK, X86::MOV64rm}}};  break;
1377 
1378     // MOVZX
1379     case X86::MOVZX16rm8:  I = {1, false, {{NOCHECK, X86::MOV16ri}}};  break;
1380     case X86::MOVZX32rm8:  I = {1, false, {{NOCHECK, X86::MOV32ri}}};  break;
1381     case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}};  break;
1382 
1383     // CMP
1384     case X86::CMP8rm:      I = {1, false, {{NOCHECK, X86::CMP8ri}}};   break;
1385     case X86::CMP16rm:     I = {2, false, {{CHECK8,  X86::CMP16ri8},
1386                                            {NOCHECK, X86::CMP16ri}}};  break;
1387     case X86::CMP32rm:     I = {4, false, {{CHECK8,  X86::CMP32ri8},
1388                                            {NOCHECK, X86::CMP32ri}}};  break;
1389     case X86::CMP64rm:     I = {8, false, {{CHECK8,  X86::CMP64ri8},
1390                                            {CHECK32, X86::CMP64ri32},
1391                                            {NOCHECK, X86::CMP64rm}}};  break;
1392 
1393     // TEST
1394     case X86::TEST8mr:     I = {1, false, {{NOCHECK, X86::TEST8ri}}};  break;
1395     case X86::TEST16mr:    I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
1396     case X86::TEST32mr:    I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
1397     case X86::TEST64mr:    I = {8, false, {{CHECK32, X86::TEST64ri32},
1398                                            {NOCHECK, X86::TEST64mr}}}; break;
1399 
1400     // ADD
1401     case X86::ADD8rm:      I = {1, true,  {{NOCHECK, X86::ADD8ri}}};   break;
1402     case X86::ADD16rm:     I = {2, true,  {{CHECK8,  X86::ADD16ri8},
1403                                            {NOCHECK, X86::ADD16ri}}};  break;
1404     case X86::ADD32rm:     I = {4, true,  {{CHECK8,  X86::ADD32ri8},
1405                                            {NOCHECK, X86::ADD32ri}}};  break;
1406     case X86::ADD64rm:     I = {8, true,  {{CHECK8,  X86::ADD64ri8},
1407                                            {CHECK32, X86::ADD64ri32},
1408                                            {NOCHECK, X86::ADD64rm}}};  break;
1409 
1410     // SUB
1411     case X86::SUB8rm:      I = {1, true,  {{NOCHECK, X86::SUB8ri}}};   break;
1412     case X86::SUB16rm:     I = {2, true,  {{CHECK8,  X86::SUB16ri8},
1413                                            {NOCHECK, X86::SUB16ri}}};  break;
1414     case X86::SUB32rm:     I = {4, true,  {{CHECK8,  X86::SUB32ri8},
1415                                            {NOCHECK, X86::SUB32ri}}};  break;
1416     case X86::SUB64rm:     I = {8, true,  {{CHECK8,  X86::SUB64ri8},
1417                                            {CHECK32, X86::SUB64ri32},
1418                                            {NOCHECK, X86::SUB64rm}}};  break;
1419 
1420     // AND
1421     case X86::AND8rm:      I = {1, true,  {{NOCHECK, X86::AND8ri}}};   break;
1422     case X86::AND16rm:     I = {2, true,  {{CHECK8,  X86::AND16ri8},
1423                                            {NOCHECK, X86::AND16ri}}};  break;
1424     case X86::AND32rm:     I = {4, true,  {{CHECK8,  X86::AND32ri8},
1425                                            {NOCHECK, X86::AND32ri}}};  break;
1426     case X86::AND64rm:     I = {8, true,  {{CHECK8,  X86::AND64ri8},
1427                                            {CHECK32, X86::AND64ri32},
1428                                            {NOCHECK, X86::AND64rm}}};  break;
1429 
1430     // OR
1431     case X86::OR8rm:       I = {1, true,  {{NOCHECK, X86::OR8ri}}};    break;
1432     case X86::OR16rm:      I = {2, true,  {{CHECK8,  X86::OR16ri8},
1433                                            {NOCHECK, X86::OR16ri}}};   break;
1434     case X86::OR32rm:      I = {4, true,  {{CHECK8,  X86::OR32ri8},
1435                                            {NOCHECK, X86::OR32ri}}};   break;
1436     case X86::OR64rm:      I = {8, true,  {{CHECK8,  X86::OR64ri8},
1437                                            {CHECK32, X86::OR64ri32},
1438                                            {NOCHECK, X86::OR64rm}}};   break;
1439 
1440     // XOR
1441     case X86::XOR8rm:      I = {1, true,  {{NOCHECK, X86::XOR8ri}}};   break;
1442     case X86::XOR16rm:     I = {2, true,  {{CHECK8,  X86::XOR16ri8},
1443                                            {NOCHECK, X86::XOR16ri}}};  break;
1444     case X86::XOR32rm:     I = {4, true,  {{CHECK8,  X86::XOR32ri8},
1445                                            {NOCHECK, X86::XOR32ri}}};  break;
1446     case X86::XOR64rm:     I = {8, true,  {{CHECK8,  X86::XOR64ri8},
1447                                            {CHECK32, X86::XOR64ri32},
1448                                            {NOCHECK, X86::XOR64rm}}};  break;
1449     }
1450 
1451     // Compute the immediate value.
1452     assert(Offset + I.DataSize <= ConstantData.size() &&
1453            "invalid offset for given constant data");
1454     int64_t ImmVal =
1455         DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize);
1456 
1457     // Compute the new opcode.
1458     unsigned NewOpcode = 0;
1459     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1460       NewOpcode = Check.second;
1461       if (Check.first == NOCHECK)
1462         break;
1463       if (Check.first == CHECK8 && isInt<8>(ImmVal))
1464         break;
1465       if (Check.first == CHECK32 && isInt<32>(ImmVal))
1466         break;
1467     }
1468     if (NewOpcode == Inst.getOpcode())
1469       return false;
1470 
1471     // Modify the instruction.
1472     MCOperand ImmOp = MCOperand::createImm(ImmVal);
1473     uint32_t TargetOpNum = 0;
1474     // Test instruction does not follow the regular pattern of putting the
1475     // memory reference of a load (5 MCOperands) last in the list of operands.
1476     // Since it is not modifying the register operand, it is not treated as
1477     // a destination operand and it is not the first operand as it is in the
1478     // other instructions we treat here.
1479     if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1480         NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1481       TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1482 
1483     MCOperand TargetOp = Inst.getOperand(TargetOpNum);
1484     Inst.clear();
1485     Inst.setOpcode(NewOpcode);
1486     Inst.addOperand(TargetOp);
1487     if (I.HasLHS)
1488       Inst.addOperand(TargetOp);
1489     Inst.addOperand(ImmOp);
1490 
1491     return true;
1492   }
1493 
1494   /// TODO: this implementation currently works for the most common opcodes that
1495   /// load from memory. It can be extended to work with memory store opcodes as
1496   /// well as more memory load opcodes.
1497   bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1498     unsigned NewOpcode;
1499 
1500     switch (Inst.getOpcode()) {
1501     default: {
1502       switch (getPopSize(Inst)) {
1503       case 2:            NewOpcode = X86::MOV16rr; break;
1504       case 4:            NewOpcode = X86::MOV32rr; break;
1505       case 8:            NewOpcode = X86::MOV64rr; break;
1506       default:           return false;
1507       }
1508       break;
1509     }
1510 
1511     // MOV
1512     case X86::MOV8rm:      NewOpcode = X86::MOV8rr;   break;
1513     case X86::MOV16rm:     NewOpcode = X86::MOV16rr;  break;
1514     case X86::MOV32rm:     NewOpcode = X86::MOV32rr;  break;
1515     case X86::MOV64rm:     NewOpcode = X86::MOV64rr;  break;
1516     }
1517 
1518     // Modify the instruction.
1519     MCOperand RegOp = MCOperand::createReg(RegNum);
1520     MCOperand TargetOp = Inst.getOperand(0);
1521     Inst.clear();
1522     Inst.setOpcode(NewOpcode);
1523     Inst.addOperand(TargetOp);
1524     Inst.addOperand(RegOp);
1525 
1526     return true;
1527   }
1528 
1529   bool isRedundantMove(const MCInst &Inst) const override {
1530     switch (Inst.getOpcode()) {
1531     default:
1532       return false;
1533 
1534     // MOV
1535     case X86::MOV8rr:
1536     case X86::MOV16rr:
1537     case X86::MOV32rr:
1538     case X86::MOV64rr:
1539       break;
1540     }
1541 
1542     assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1543     return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg();
1544   }
1545 
1546   bool requiresAlignedAddress(const MCInst &Inst) const override {
1547     const MCInstrDesc &Desc = Info->get(Inst.getOpcode());
1548     for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1549       const MCOperandInfo &Op = Desc.operands()[I];
1550       if (Op.OperandType != MCOI::OPERAND_REGISTER)
1551         continue;
1552       if (Op.RegClass == X86::VR128RegClassID)
1553         return true;
1554     }
1555     return false;
1556   }
1557 
1558   bool convertJmpToTailCall(MCInst &Inst) override {
1559     if (isTailCall(Inst))
1560       return false;
1561 
1562     int NewOpcode;
1563     switch (Inst.getOpcode()) {
1564     default:
1565       return false;
1566     case X86::JMP_1:
1567     case X86::JMP_2:
1568     case X86::JMP_4:
1569       NewOpcode = X86::JMP_4;
1570       break;
1571     case X86::JMP16m:
1572     case X86::JMP32m:
1573     case X86::JMP64m:
1574       NewOpcode = X86::JMP32m;
1575       break;
1576     case X86::JMP16r:
1577     case X86::JMP32r:
1578     case X86::JMP64r:
1579       NewOpcode = X86::JMP32r;
1580       break;
1581     }
1582 
1583     Inst.setOpcode(NewOpcode);
1584     setTailCall(Inst);
1585     return true;
1586   }
1587 
1588   bool convertTailCallToJmp(MCInst &Inst) override {
1589     int NewOpcode;
1590     switch (Inst.getOpcode()) {
1591     default:
1592       return false;
1593     case X86::JMP_4:
1594       NewOpcode = X86::JMP_1;
1595       break;
1596     case X86::JMP32m:
1597       NewOpcode = X86::JMP64m;
1598       break;
1599     case X86::JMP32r:
1600       NewOpcode = X86::JMP64r;
1601       break;
1602     }
1603 
1604     Inst.setOpcode(NewOpcode);
1605     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1606     clearOffset(Inst);
1607     return true;
1608   }
1609 
1610   bool convertTailCallToCall(MCInst &Inst) override {
1611     int NewOpcode;
1612     switch (Inst.getOpcode()) {
1613     default:
1614       return false;
1615     case X86::JMP_4:
1616       NewOpcode = X86::CALL64pcrel32;
1617       break;
1618     case X86::JMP32m:
1619       NewOpcode = X86::CALL64m;
1620       break;
1621     case X86::JMP32r:
1622       NewOpcode = X86::CALL64r;
1623       break;
1624     }
1625 
1626     Inst.setOpcode(NewOpcode);
1627     removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1628     return true;
1629   }
1630 
1631   bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation,
1632                                  MCContext *Ctx) override {
1633     assert((Inst.getOpcode() == X86::CALL64pcrel32 ||
1634             (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) &&
1635            "64-bit direct (tail) call instruction expected");
1636     const auto NewOpcode =
1637         (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1638     Inst.setOpcode(NewOpcode);
1639 
1640     // Replace the first operand and preserve auxiliary operands of
1641     // the instruction.
1642     Inst.erase(Inst.begin());
1643     Inst.insert(Inst.begin(),
1644                 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
1645     Inst.insert(Inst.begin(),
1646                 MCOperand::createExpr(                  // Displacement
1647                     MCSymbolRefExpr::create(TargetLocation,
1648                                             MCSymbolRefExpr::VK_None, *Ctx)));
1649     Inst.insert(Inst.begin(),
1650                 MCOperand::createReg(X86::NoRegister)); // IndexReg
1651     Inst.insert(Inst.begin(),
1652                 MCOperand::createImm(1));               // ScaleAmt
1653     Inst.insert(Inst.begin(),
1654                 MCOperand::createReg(X86::RIP));        // BaseReg
1655 
1656     return true;
1657   }
1658 
1659   void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1660     bool IsTailCall = isTailCall(Inst);
1661     if (IsTailCall)
1662       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1663     if (Inst.getOpcode() == X86::CALL64m ||
1664         (Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1665       Inst.setOpcode(X86::MOV64rm);
1666       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1667       return;
1668     }
1669     if (Inst.getOpcode() == X86::CALL64r ||
1670         (Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1671       Inst.setOpcode(X86::MOV64rr);
1672       Inst.insert(Inst.begin(), MCOperand::createReg(Reg));
1673       return;
1674     }
1675     LLVM_DEBUG(Inst.dump());
1676     llvm_unreachable("not implemented");
1677   }
1678 
1679   bool shortenInstruction(MCInst &Inst,
1680                           const MCSubtargetInfo &STI) const override {
1681     unsigned OldOpcode = Inst.getOpcode();
1682     unsigned NewOpcode = OldOpcode;
1683 
1684     int MemOpNo = getMemoryOperandNo(Inst);
1685 
1686     // Check and remove redundant Address-Size override prefix.
1687     if (opts::X86StripRedundantAddressSize) {
1688       uint64_t TSFlags = Info->get(OldOpcode).TSFlags;
1689       unsigned Flags = Inst.getFlags();
1690 
1691       if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) &&
1692           Flags & X86::IP_HAS_AD_SIZE)
1693         Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1694     }
1695 
1696     // Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1697     // SIB byte is present, but no index is used and modrm alone should have
1698     // been enough. Converting to NoRegister effectively removes the SIB byte.
1699     if (MemOpNo >= 0) {
1700       MCOperand &IndexOp =
1701           Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1702       if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1703         IndexOp = MCOperand::createReg(X86::NoRegister);
1704     }
1705 
1706     if (isBranch(Inst)) {
1707       NewOpcode = getShortBranchOpcode(OldOpcode);
1708     } else if (OldOpcode == X86::MOV64ri) {
1709       if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1710         const int64_t Imm =
1711             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1712         if (int64_t(Imm) == int64_t(int32_t(Imm)))
1713           NewOpcode = X86::MOV64ri32;
1714       }
1715     } else {
1716       // If it's arithmetic instruction check if signed operand fits in 1 byte.
1717       const unsigned ShortOpcode = getShortArithOpcode(OldOpcode);
1718       if (ShortOpcode != OldOpcode &&
1719           Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1720         int64_t Imm =
1721             Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1722         if (int64_t(Imm) == int64_t(int8_t(Imm)))
1723           NewOpcode = ShortOpcode;
1724       }
1725     }
1726 
1727     if (NewOpcode == OldOpcode)
1728       return false;
1729 
1730     Inst.setOpcode(NewOpcode);
1731     return true;
1732   }
1733 
1734   bool
1735   convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1736                                bool AllowBasePtrStackMemOp) const override {
1737     // - Register-register moves are OK
1738     // - Stores are filtered out by opcode (no store CMOV)
1739     // - Non-stack loads are prohibited (generally unsafe)
1740     // - Stack loads are OK if AllowStackMemOp is true
1741     // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1742     if (isLoad(Inst)) {
1743       // If stack memory operands are not allowed, no loads are allowed
1744       if (!AllowStackMemOp)
1745         return false;
1746 
1747       // If stack memory operands are allowed, check if it's a load from stack
1748       bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1749       MCPhysReg Reg;
1750       int32_t SrcImm;
1751       uint16_t StackPtrReg;
1752       int64_t StackOffset;
1753       uint8_t Size;
1754       bool IsStackAccess =
1755           isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1756                         StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1757       // Prohibit non-stack-based loads
1758       if (!IsStackAccess)
1759         return false;
1760       // If stack memory operands are allowed, check if it's RBP-based
1761       if (!AllowBasePtrStackMemOp &&
1762           RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg))
1763         return false;
1764     }
1765 
1766     unsigned NewOpcode = 0;
1767     switch (Inst.getOpcode()) {
1768     case X86::MOV16rr:
1769       NewOpcode = X86::CMOV16rr;
1770       break;
1771     case X86::MOV16rm:
1772       NewOpcode = X86::CMOV16rm;
1773       break;
1774     case X86::MOV32rr:
1775       NewOpcode = X86::CMOV32rr;
1776       break;
1777     case X86::MOV32rm:
1778       NewOpcode = X86::CMOV32rm;
1779       break;
1780     case X86::MOV64rr:
1781       NewOpcode = X86::CMOV64rr;
1782       break;
1783     case X86::MOV64rm:
1784       NewOpcode = X86::CMOV64rm;
1785       break;
1786     default:
1787       return false;
1788     }
1789     Inst.setOpcode(NewOpcode);
1790     // Insert CC at the end of prime operands, before annotations
1791     Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1792                 MCOperand::createImm(CC));
1793     // CMOV is a 3-operand MCInst, so duplicate the destination as src1
1794     Inst.insert(Inst.begin(), Inst.getOperand(0));
1795     return true;
1796   }
1797 
1798   bool lowerTailCall(MCInst &Inst) override {
1799     if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1800       Inst.setOpcode(X86::JMP_1);
1801       removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);
1802       return true;
1803     }
1804     return false;
1805   }
1806 
1807   const MCSymbol *getTargetSymbol(const MCInst &Inst,
1808                                   unsigned OpNum = 0) const override {
1809     if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1810       return nullptr;
1811 
1812     const MCOperand &Op = Inst.getOperand(OpNum);
1813     if (!Op.isExpr())
1814       return nullptr;
1815 
1816     auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
1817     if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None)
1818       return nullptr;
1819 
1820     return &SymExpr->getSymbol();
1821   }
1822 
1823   // This is the same as the base class, but since we are overriding one of
1824   // getTargetSymbol's signatures above, we need to override all of them.
1825   const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1826     return &cast<const MCSymbolRefExpr>(Expr)->getSymbol();
1827   }
1828 
1829   bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1830                      const MCSymbol *&TBB, const MCSymbol *&FBB,
1831                      MCInst *&CondBranch,
1832                      MCInst *&UncondBranch) const override {
1833     auto I = End;
1834 
1835     // Bottom-up analysis
1836     while (I != Begin) {
1837       --I;
1838 
1839       // Ignore nops and CFIs
1840       if (isPseudo(*I))
1841         continue;
1842 
1843       // Stop when we find the first non-terminator
1844       if (!isTerminator(*I))
1845         break;
1846 
1847       if (!isBranch(*I))
1848         break;
1849 
1850       // Handle unconditional branches.
1851       if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1852            I->getOpcode() == X86::JMP_4) &&
1853           !isTailCall(*I)) {
1854         // If any code was seen after this unconditional branch, we've seen
1855         // unreachable code. Ignore them.
1856         CondBranch = nullptr;
1857         UncondBranch = &*I;
1858         const MCSymbol *Sym = getTargetSymbol(*I);
1859         assert(Sym != nullptr &&
1860                "Couldn't extract BB symbol from jump operand");
1861         TBB = Sym;
1862         continue;
1863       }
1864 
1865       // Handle conditional branches and ignore indirect branches
1866       if (!isUnsupportedBranch(I->getOpcode()) &&
1867           getCondCode(*I) == X86::COND_INVALID) {
1868         // Indirect branch
1869         return false;
1870       }
1871 
1872       if (CondBranch == nullptr) {
1873         const MCSymbol *TargetBB = getTargetSymbol(*I);
1874         if (TargetBB == nullptr) {
1875           // Unrecognized branch target
1876           return false;
1877         }
1878         FBB = TBB;
1879         TBB = TargetBB;
1880         CondBranch = &*I;
1881         continue;
1882       }
1883 
1884       llvm_unreachable("multiple conditional branches in one BB");
1885     }
1886     return true;
1887   }
1888 
1889   template <typename Itr>
1890   std::pair<IndirectBranchType, MCInst *>
1891   analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
1892     // Analyze PIC-style jump table code template:
1893     //
1894     //    lea PIC_JUMP_TABLE(%rip), {%r1|%r2}     <- MemLocInstr
1895     //    mov ({%r1|%r2}, %index, 4), {%r2|%r1}
1896     //    add %r2, %r1
1897     //    jmp *%r1
1898     //
1899     // (with any irrelevant instructions in-between)
1900     //
1901     // When we call this helper we've already determined %r1 and %r2, and
1902     // reverse instruction iterator \p II is pointing to the ADD instruction.
1903     //
1904     // PIC jump table looks like following:
1905     //
1906     //   JT:  ----------
1907     //    E1:| L1 - JT  |
1908     //       |----------|
1909     //    E2:| L2 - JT  |
1910     //       |----------|
1911     //       |          |
1912     //          ......
1913     //    En:| Ln - JT  |
1914     //        ----------
1915     //
1916     // Where L1, L2, ..., Ln represent labels in the function.
1917     //
1918     // The actual relocations in the table will be of the form:
1919     //
1920     //   Ln - JT
1921     //    = (Ln - En) + (En - JT)
1922     //    = R_X86_64_PC32(Ln) + En - JT
1923     //    = R_X86_64_PC32(Ln + offsetof(En))
1924     //
1925     LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
1926     MCInst *MemLocInstr = nullptr;
1927     const MCInst *MovInstr = nullptr;
1928     while (++II != IE) {
1929       MCInst &Instr = *II;
1930       const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode());
1931       if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) &&
1932           !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) {
1933         // Ignore instructions that don't affect R1, R2 registers.
1934         continue;
1935       }
1936       if (!MovInstr) {
1937         // Expect to see MOV instruction.
1938         if (!isMOVSX64rm32(Instr)) {
1939           LLVM_DEBUG(dbgs() << "MOV instruction expected.\n");
1940           break;
1941         }
1942 
1943         // Check if it's setting %r1 or %r2. In canonical form it sets %r2.
1944         // If it sets %r1 - rename the registers so we have to only check
1945         // a single form.
1946         unsigned MovDestReg = Instr.getOperand(0).getReg();
1947         if (MovDestReg != R2)
1948           std::swap(R1, R2);
1949         if (MovDestReg != R2) {
1950           LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n");
1951           break;
1952         }
1953 
1954         // Verify operands for MOV.
1955         std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
1956         if (!MO)
1957           break;
1958         if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 ||
1959             MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 ||
1960             MO->SegRegNum != X86::NoRegister)
1961           break;
1962         MovInstr = &Instr;
1963       } else {
1964         if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo))
1965           continue;
1966         if (!isLEA64r(Instr)) {
1967           LLVM_DEBUG(dbgs() << "LEA instruction expected\n");
1968           break;
1969         }
1970         if (Instr.getOperand(0).getReg() != R1) {
1971           LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n");
1972           break;
1973         }
1974 
1975         // Verify operands for LEA.
1976         std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
1977         if (!MO)
1978           break;
1979         if (MO->BaseRegNum != RegInfo->getProgramCounter() ||
1980             MO->IndexRegNum != X86::NoRegister ||
1981             MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr)
1982           break;
1983         MemLocInstr = &Instr;
1984         break;
1985       }
1986     }
1987 
1988     if (!MemLocInstr)
1989       return std::make_pair(IndirectBranchType::UNKNOWN, nullptr);
1990 
1991     LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
1992     return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
1993                           MemLocInstr);
1994   }
1995 
1996   IndirectBranchType analyzeIndirectBranch(
1997       MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
1998       const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
1999       unsigned &IndexRegNumOut, int64_t &DispValueOut,
2000       const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
2001     // Try to find a (base) memory location from where the address for
2002     // the indirect branch is loaded. For X86-64 the memory will be specified
2003     // in the following format:
2004     //
2005     //   {%rip}/{%basereg} + Imm + IndexReg * Scale
2006     //
2007     // We are interested in the cases where Scale == sizeof(uintptr_t) and
2008     // the contents of the memory are presumably an array of pointers to code.
2009     //
2010     // Normal jump table:
2011     //
2012     //    jmp *(JUMP_TABLE, %index, Scale)        <- MemLocInstr
2013     //
2014     //    or
2015     //
2016     //    mov (JUMP_TABLE, %index, Scale), %r1    <- MemLocInstr
2017     //    ...
2018     //    jmp %r1
2019     //
2020     // We handle PIC-style jump tables separately.
2021     //
2022     MemLocInstrOut = nullptr;
2023     BaseRegNumOut = X86::NoRegister;
2024     IndexRegNumOut = X86::NoRegister;
2025     DispValueOut = 0;
2026     DispExprOut = nullptr;
2027 
2028     std::reverse_iterator<InstructionIterator> II(End);
2029     std::reverse_iterator<InstructionIterator> IE(Begin);
2030 
2031     IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2032 
2033     // An instruction referencing memory used by jump instruction (directly or
2034     // via register). This location could be an array of function pointers
2035     // in case of indirect tail call, or a jump table.
2036     MCInst *MemLocInstr = nullptr;
2037 
2038     if (MCPlus::getNumPrimeOperands(Instruction) == 1) {
2039       // If the indirect jump is on register - try to detect if the
2040       // register value is loaded from a memory location.
2041       assert(Instruction.getOperand(0).isReg() && "register operand expected");
2042       const unsigned R1 = Instruction.getOperand(0).getReg();
2043       // Check if one of the previous instructions defines the jump-on register.
2044       for (auto PrevII = II; PrevII != IE; ++PrevII) {
2045         MCInst &PrevInstr = *PrevII;
2046         const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode());
2047 
2048         if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo))
2049           continue;
2050 
2051         if (isMoveMem2Reg(PrevInstr)) {
2052           MemLocInstr = &PrevInstr;
2053           break;
2054         }
2055         if (isADD64rr(PrevInstr)) {
2056           unsigned R2 = PrevInstr.getOperand(2).getReg();
2057           if (R1 == R2)
2058             return IndirectBranchType::UNKNOWN;
2059           std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
2060           break;
2061         }
2062         return IndirectBranchType::UNKNOWN;
2063       }
2064       if (!MemLocInstr) {
2065         // No definition seen for the register in this function so far. Could be
2066         // an input parameter - which means it is an external code reference.
2067         // It also could be that the definition happens to be in the code that
2068         // we haven't processed yet. Since we have to be conservative, return
2069         // as UNKNOWN case.
2070         return IndirectBranchType::UNKNOWN;
2071       }
2072     } else {
2073       MemLocInstr = &Instruction;
2074     }
2075 
2076     const MCRegister RIPRegister = RegInfo->getProgramCounter();
2077 
2078     // Analyze the memory location.
2079     std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr);
2080     if (!MO)
2081       return IndirectBranchType::UNKNOWN;
2082 
2083     BaseRegNumOut = MO->BaseRegNum;
2084     IndexRegNumOut = MO->IndexRegNum;
2085     DispValueOut = MO->DispImm;
2086     DispExprOut = MO->DispExpr;
2087 
2088     if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) ||
2089         MO->SegRegNum != X86::NoRegister)
2090       return IndirectBranchType::UNKNOWN;
2091 
2092     if (MemLocInstr == &Instruction &&
2093         (!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) {
2094       MemLocInstrOut = MemLocInstr;
2095       return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2096     }
2097 
2098     if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2099         (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister))
2100       return IndirectBranchType::UNKNOWN;
2101 
2102     if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2103         MO->ScaleImm != PtrSize)
2104       return IndirectBranchType::UNKNOWN;
2105 
2106     MemLocInstrOut = MemLocInstr;
2107 
2108     return Type;
2109   }
2110 
2111   /// Analyze a callsite to see if it could be a virtual method call.  This only
2112   /// checks to see if the overall pattern is satisfied, it does not guarantee
2113   /// that the callsite is a true virtual method call.
2114   /// The format of virtual method calls that are recognized is one of the
2115   /// following:
2116   ///
2117   ///  Form 1: (found in debug code)
2118   ///    add METHOD_OFFSET, %VtableReg
2119   ///    mov (%VtableReg), %MethodReg
2120   ///    ...
2121   ///    call or jmp *%MethodReg
2122   ///
2123   ///  Form 2:
2124   ///    mov METHOD_OFFSET(%VtableReg), %MethodReg
2125   ///    ...
2126   ///    call or jmp *%MethodReg
2127   ///
2128   ///  Form 3:
2129   ///    ...
2130   ///    call or jmp *METHOD_OFFSET(%VtableReg)
2131   ///
2132   bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2133                                 InstructionIterator ForwardEnd,
2134                                 std::vector<MCInst *> &MethodFetchInsns,
2135                                 unsigned &VtableRegNum, unsigned &MethodRegNum,
2136                                 uint64_t &MethodOffset) const override {
2137     VtableRegNum = X86::NoRegister;
2138     MethodRegNum = X86::NoRegister;
2139     MethodOffset = 0;
2140 
2141     std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2142     std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2143 
2144     MCInst &CallInst = *Itr++;
2145     assert(isIndirectBranch(CallInst) || isCall(CallInst));
2146 
2147     // The call can just be jmp offset(reg)
2148     if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) {
2149       if (!MO->DispExpr && MO->BaseRegNum != X86::RIP &&
2150           MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) {
2151         MethodRegNum = MO->BaseRegNum;
2152         if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister &&
2153             MO->SegRegNum == X86::NoRegister) {
2154           VtableRegNum = MethodRegNum;
2155           MethodOffset = MO->DispImm;
2156           MethodFetchInsns.push_back(&CallInst);
2157           return true;
2158         }
2159       }
2160       return false;
2161     }
2162     if (CallInst.getOperand(0).isReg())
2163       MethodRegNum = CallInst.getOperand(0).getReg();
2164     else
2165       return false;
2166 
2167     if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2168       VtableRegNum = X86::NoRegister;
2169       MethodRegNum = X86::NoRegister;
2170       return false;
2171     }
2172 
2173     // find load from vtable, this may or may not include the method offset
2174     while (Itr != End) {
2175       MCInst &CurInst = *Itr++;
2176       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2177       if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) {
2178         if (!isLoad(CurInst))
2179           return false;
2180         if (std::optional<X86MemOperand> MO =
2181                 evaluateX86MemoryOperand(CurInst)) {
2182           if (!MO->DispExpr && MO->ScaleImm == 1 &&
2183               MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP &&
2184               MO->BaseRegNum != X86::NoRegister &&
2185               MO->IndexRegNum == X86::NoRegister &&
2186               MO->SegRegNum == X86::NoRegister && MO->BaseRegNum != X86::RIP) {
2187             VtableRegNum = MO->BaseRegNum;
2188             MethodOffset = MO->DispImm;
2189             MethodFetchInsns.push_back(&CurInst);
2190             if (MethodOffset != 0)
2191               return true;
2192             break;
2193           }
2194         }
2195         return false;
2196       }
2197     }
2198 
2199     if (!VtableRegNum)
2200       return false;
2201 
2202     // look for any adds affecting the method register.
2203     while (Itr != End) {
2204       MCInst &CurInst = *Itr++;
2205       const MCInstrDesc &Desc = Info->get(CurInst.getOpcode());
2206       if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) {
2207         if (isADDri(CurInst)) {
2208           assert(!MethodOffset);
2209           MethodOffset = CurInst.getOperand(2).getImm();
2210           MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst);
2211           break;
2212         }
2213       }
2214     }
2215 
2216     return true;
2217   }
2218 
2219   bool createStackPointerIncrement(MCInst &Inst, int Size,
2220                                    bool NoFlagsClobber) const override {
2221     if (NoFlagsClobber) {
2222       Inst.setOpcode(X86::LEA64r);
2223       Inst.clear();
2224       Inst.addOperand(MCOperand::createReg(X86::RSP));
2225       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2226       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2227       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2228       Inst.addOperand(MCOperand::createImm(-Size));           // Displacement
2229       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2230       return true;
2231     }
2232     Inst.setOpcode(X86::SUB64ri8);
2233     Inst.clear();
2234     Inst.addOperand(MCOperand::createReg(X86::RSP));
2235     Inst.addOperand(MCOperand::createReg(X86::RSP));
2236     Inst.addOperand(MCOperand::createImm(Size));
2237     return true;
2238   }
2239 
2240   bool createStackPointerDecrement(MCInst &Inst, int Size,
2241                                    bool NoFlagsClobber) const override {
2242     if (NoFlagsClobber) {
2243       Inst.setOpcode(X86::LEA64r);
2244       Inst.clear();
2245       Inst.addOperand(MCOperand::createReg(X86::RSP));
2246       Inst.addOperand(MCOperand::createReg(X86::RSP));        // BaseReg
2247       Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2248       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2249       Inst.addOperand(MCOperand::createImm(Size));            // Displacement
2250       Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2251       return true;
2252     }
2253     Inst.setOpcode(X86::ADD64ri8);
2254     Inst.clear();
2255     Inst.addOperand(MCOperand::createReg(X86::RSP));
2256     Inst.addOperand(MCOperand::createReg(X86::RSP));
2257     Inst.addOperand(MCOperand::createImm(Size));
2258     return true;
2259   }
2260 
2261   bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2262                          const MCPhysReg &SrcReg, int Size) const override {
2263     unsigned NewOpcode;
2264     switch (Size) {
2265     default:
2266       return false;
2267     case 2:      NewOpcode = X86::MOV16mr; break;
2268     case 4:      NewOpcode = X86::MOV32mr; break;
2269     case 8:      NewOpcode = X86::MOV64mr; break;
2270     }
2271     Inst.setOpcode(NewOpcode);
2272     Inst.clear();
2273     Inst.addOperand(MCOperand::createReg(StackReg));        // BaseReg
2274     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2275     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2276     Inst.addOperand(MCOperand::createImm(Offset));          // Displacement
2277     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2278     Inst.addOperand(MCOperand::createReg(SrcReg));
2279     return true;
2280   }
2281 
2282   bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2283                               int Offset, const MCPhysReg &DstReg,
2284                               int Size) const override {
2285     return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2286                       Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2287                       DstReg, Size);
2288   }
2289 
2290   bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2291                   const MCPhysReg &IndexReg, int64_t Offset,
2292                   const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2293                   const MCPhysReg &DstReg, int Size) const override {
2294     unsigned NewOpcode;
2295     switch (Size) {
2296     default:
2297       return false;
2298     case 2:      NewOpcode = X86::MOV16rm; break;
2299     case 4:      NewOpcode = X86::MOV32rm; break;
2300     case 8:      NewOpcode = X86::MOV64rm; break;
2301     }
2302     Inst.setOpcode(NewOpcode);
2303     Inst.clear();
2304     Inst.addOperand(MCOperand::createReg(DstReg));
2305     Inst.addOperand(MCOperand::createReg(BaseReg));
2306     Inst.addOperand(MCOperand::createImm(Scale));
2307     Inst.addOperand(MCOperand::createReg(IndexReg));
2308     if (OffsetExpr)
2309       Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement
2310     else
2311       Inst.addOperand(MCOperand::createImm(Offset)); // Displacement
2312     Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg
2313     return true;
2314   }
2315 
2316   void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest,
2317                            uint32_t Imm) const override {
2318     Inst.setOpcode(X86::MOV64ri32);
2319     Inst.clear();
2320     Inst.addOperand(MCOperand::createReg(Dest));
2321     Inst.addOperand(MCOperand::createImm(Imm));
2322   }
2323 
2324   bool createIncMemory(MCInst &Inst, const MCSymbol *Target,
2325                        MCContext *Ctx) const override {
2326 
2327     Inst.setOpcode(X86::LOCK_INC64m);
2328     Inst.clear();
2329     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
2330     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
2331     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2332 
2333     Inst.addOperand(MCOperand::createExpr(
2334         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None,
2335                                 *Ctx)));                    // Displacement
2336     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2337     return true;
2338   }
2339 
2340   bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2341                         const MCOperand &BaseReg, const MCOperand &Scale,
2342                         const MCOperand &IndexReg, const MCOperand &Offset,
2343                         const MCOperand &TmpReg) const override {
2344     // The code fragment we emit here is:
2345     //
2346     //  mov32 (%base, %index, scale), %tmpreg
2347     //  ijmp *(%tmpreg)
2348     //
2349     MCInst IJmp;
2350     IJmp.setOpcode(X86::JMP64r);
2351     IJmp.addOperand(TmpReg);
2352 
2353     MCInst Load;
2354     Load.setOpcode(X86::MOV32rm);
2355     Load.addOperand(TmpReg);
2356     Load.addOperand(BaseReg);
2357     Load.addOperand(Scale);
2358     Load.addOperand(IndexReg);
2359     Load.addOperand(Offset);
2360     Load.addOperand(MCOperand::createReg(X86::NoRegister));
2361 
2362     Insts.push_back(Load);
2363     Insts.push_back(IJmp);
2364     return true;
2365   }
2366 
2367   bool createNoop(MCInst &Inst) const override {
2368     Inst.setOpcode(X86::NOOP);
2369     return true;
2370   }
2371 
2372   bool createReturn(MCInst &Inst) const override {
2373     Inst.setOpcode(X86::RET64);
2374     return true;
2375   }
2376 
2377   InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2378     InstructionListType Code;
2379     if (ReturnEnd)
2380       Code.emplace_back(MCInstBuilder(X86::LEA64r)
2381                             .addReg(X86::RAX)
2382                             .addReg(X86::RDI)
2383                             .addImm(1)
2384                             .addReg(X86::RDX)
2385                             .addImm(0)
2386                             .addReg(X86::NoRegister));
2387     else
2388       Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2389                             .addReg(X86::RAX)
2390                             .addReg(X86::RDI));
2391 
2392     Code.emplace_back(MCInstBuilder(X86::MOV32rr)
2393                           .addReg(X86::ECX)
2394                           .addReg(X86::EDX));
2395     Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64));
2396 
2397     return Code;
2398   }
2399 
2400   InstructionListType createOneByteMemcpy() const override {
2401     InstructionListType Code;
2402     Code.emplace_back(MCInstBuilder(X86::MOV8rm)
2403                           .addReg(X86::CL)
2404                           .addReg(X86::RSI)
2405                           .addImm(0)
2406                           .addReg(X86::NoRegister)
2407                           .addImm(0)
2408                           .addReg(X86::NoRegister));
2409     Code.emplace_back(MCInstBuilder(X86::MOV8mr)
2410                           .addReg(X86::RDI)
2411                           .addImm(0)
2412                           .addReg(X86::NoRegister)
2413                           .addImm(0)
2414                           .addReg(X86::NoRegister)
2415                           .addReg(X86::CL));
2416     Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2417                           .addReg(X86::RAX)
2418                           .addReg(X86::RDI));
2419     return Code;
2420   }
2421 
2422   InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2423                                   const MCSymbol *Target,
2424                                   MCContext *Ctx) const override {
2425     InstructionListType Code;
2426     Code.emplace_back(MCInstBuilder(X86::CMP64ri8)
2427                           .addReg(RegNo)
2428                           .addImm(Imm));
2429     Code.emplace_back(MCInstBuilder(X86::JCC_1)
2430                           .addExpr(MCSymbolRefExpr::create(
2431                               Target, MCSymbolRefExpr::VK_None, *Ctx))
2432                           .addImm(X86::COND_E));
2433     return Code;
2434   }
2435 
2436   std::optional<Relocation>
2437   createRelocation(const MCFixup &Fixup,
2438                    const MCAsmBackend &MAB) const override {
2439     const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind());
2440 
2441     assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2442     const uint64_t RelOffset = Fixup.getOffset();
2443 
2444     uint64_t RelType;
2445     if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2446       switch (FKI.TargetSize) {
2447       default:
2448         return std::nullopt;
2449       case  8: RelType = ELF::R_X86_64_PC8; break;
2450       case 16: RelType = ELF::R_X86_64_PC16; break;
2451       case 32: RelType = ELF::R_X86_64_PC32; break;
2452       case 64: RelType = ELF::R_X86_64_PC64; break;
2453       }
2454     } else {
2455       switch (FKI.TargetSize) {
2456       default:
2457         return std::nullopt;
2458       case  8: RelType = ELF::R_X86_64_8; break;
2459       case 16: RelType = ELF::R_X86_64_16; break;
2460       case 32: RelType = ELF::R_X86_64_32; break;
2461       case 64: RelType = ELF::R_X86_64_64; break;
2462       }
2463     }
2464 
2465     // Extract a symbol and an addend out of the fixup value expression.
2466     //
2467     // Only the following limited expression types are supported:
2468     //   Symbol + Addend
2469     //   Symbol
2470     uint64_t Addend = 0;
2471     MCSymbol *Symbol = nullptr;
2472     const MCExpr *ValueExpr = Fixup.getValue();
2473     if (ValueExpr->getKind() == MCExpr::Binary) {
2474       const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr);
2475       assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add &&
2476              "unexpected binary expression");
2477       const MCExpr *LHS = BinaryExpr->getLHS();
2478       assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS");
2479       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS));
2480       const MCExpr *RHS = BinaryExpr->getRHS();
2481       assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS");
2482       Addend = cast<MCConstantExpr>(RHS)->getValue();
2483     } else {
2484       assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value");
2485       Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr));
2486     }
2487 
2488     return Relocation({RelOffset, Symbol, RelType, Addend, 0});
2489   }
2490 
2491   bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2492                                int64_t Addend, MCContext *Ctx, int64_t &Value,
2493                                uint64_t RelType) const override {
2494     unsigned ImmOpNo = -1U;
2495 
2496     for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2497          ++Index) {
2498       if (Inst.getOperand(Index).isImm()) {
2499         ImmOpNo = Index;
2500         // TODO: this is a bit hacky.  It finds the correct operand by
2501         // searching for a specific immediate value.  If no value is
2502         // provided it defaults to the last immediate operand found.
2503         // This could lead to unexpected results if the instruction
2504         // has more than one immediate with the same value.
2505         if (Inst.getOperand(ImmOpNo).getImm() == Value)
2506           break;
2507       }
2508     }
2509 
2510     if (ImmOpNo == -1U)
2511       return false;
2512 
2513     Value = Inst.getOperand(ImmOpNo).getImm();
2514 
2515     setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType);
2516 
2517     return true;
2518   }
2519 
2520   bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2521                          int64_t Imm) const override {
2522 
2523     enum CheckSignExt : uint8_t {
2524       NOCHECK = 0,
2525       CHECK8,
2526       CHECK32,
2527     };
2528 
2529     using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2530     struct InstInfo {
2531       // Size in bytes that Inst loads from memory.
2532       uint8_t DataSize;
2533 
2534       // True when the target operand has to be duplicated because the opcode
2535       // expects a LHS operand.
2536       bool HasLHS;
2537 
2538       // List of checks and corresponding opcodes to be used. We try to use the
2539       // smallest possible immediate value when various sizes are available,
2540       // hence we may need to check whether a larger constant fits in a smaller
2541       // immediate.
2542       CheckList Checks;
2543     };
2544 
2545     InstInfo I;
2546 
2547     switch (Inst.getOpcode()) {
2548     default: {
2549       switch (getPushSize(Inst)) {
2550 
2551       case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break;
2552       case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break;
2553       case 8: I = {8, false, {{CHECK8, X86::PUSH64i8},
2554                               {CHECK32, X86::PUSH64i32},
2555                               {NOCHECK, Inst.getOpcode()}}}; break;
2556       default: return false;
2557       }
2558       break;
2559     }
2560 
2561     // MOV
2562     case X86::MOV8rr:       I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
2563     case X86::MOV16rr:      I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
2564     case X86::MOV32rr:      I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
2565     case X86::MOV64rr:      I = {8, false, {{CHECK32, X86::MOV64ri32},
2566                                             {NOCHECK, X86::MOV64ri}}}; break;
2567 
2568     case X86::MOV8mr:       I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break;
2569     case X86::MOV16mr:      I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break;
2570     case X86::MOV32mr:      I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break;
2571     case X86::MOV64mr:      I = {8, false, {{CHECK32, X86::MOV64mi32},
2572                                             {NOCHECK, X86::MOV64mr}}}; break;
2573 
2574     // MOVZX
2575     case X86::MOVZX16rr8:   I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
2576     case X86::MOVZX32rr8:   I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
2577     case X86::MOVZX32rr16:  I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
2578 
2579     // CMP
2580     case X86::CMP8rr:       I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
2581     case X86::CMP16rr:      I = {2, false, {{CHECK8, X86::CMP16ri8},
2582                                             {NOCHECK, X86::CMP16ri}}}; break;
2583     case X86::CMP32rr:      I = {4, false, {{CHECK8, X86::CMP32ri8},
2584                                             {NOCHECK, X86::CMP32ri}}}; break;
2585     case X86::CMP64rr:      I = {8, false, {{CHECK8, X86::CMP64ri8},
2586                                             {CHECK32, X86::CMP64ri32},
2587                                             {NOCHECK, X86::CMP64rr}}}; break;
2588 
2589     // TEST
2590     case X86::TEST8rr:      I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
2591     case X86::TEST16rr:     I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
2592     case X86::TEST32rr:     I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
2593     case X86::TEST64rr:     I = {8, false, {{CHECK32, X86::TEST64ri32},
2594                                             {NOCHECK, X86::TEST64rr}}}; break;
2595 
2596     // ADD
2597     case X86::ADD8rr:       I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
2598     case X86::ADD16rr:      I = {2, true, {{CHECK8, X86::ADD16ri8},
2599                                            {NOCHECK, X86::ADD16ri}}}; break;
2600     case X86::ADD32rr:      I = {4, true, {{CHECK8, X86::ADD32ri8},
2601                                            {NOCHECK, X86::ADD32ri}}}; break;
2602     case X86::ADD64rr:      I = {8, true, {{CHECK8, X86::ADD64ri8},
2603                                            {CHECK32, X86::ADD64ri32},
2604                                            {NOCHECK, X86::ADD64rr}}}; break;
2605 
2606     // SUB
2607     case X86::SUB8rr:       I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
2608     case X86::SUB16rr:      I = {2, true, {{CHECK8, X86::SUB16ri8},
2609                                            {NOCHECK, X86::SUB16ri}}}; break;
2610     case X86::SUB32rr:      I = {4, true, {{CHECK8, X86::SUB32ri8},
2611                                            {NOCHECK, X86::SUB32ri}}}; break;
2612     case X86::SUB64rr:      I = {8, true, {{CHECK8, X86::SUB64ri8},
2613                                            {CHECK32, X86::SUB64ri32},
2614                                            {NOCHECK, X86::SUB64rr}}}; break;
2615 
2616     // AND
2617     case X86::AND8rr:       I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
2618     case X86::AND16rr:      I = {2, true, {{CHECK8, X86::AND16ri8},
2619                                            {NOCHECK, X86::AND16ri}}}; break;
2620     case X86::AND32rr:      I = {4, true, {{CHECK8, X86::AND32ri8},
2621                                            {NOCHECK, X86::AND32ri}}}; break;
2622     case X86::AND64rr:      I = {8, true, {{CHECK8, X86::AND64ri8},
2623                                            {CHECK32, X86::AND64ri32},
2624                                            {NOCHECK, X86::AND64rr}}}; break;
2625 
2626     // OR
2627     case X86::OR8rr:        I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
2628     case X86::OR16rr:       I = {2, true, {{CHECK8, X86::OR16ri8},
2629                                            {NOCHECK, X86::OR16ri}}}; break;
2630     case X86::OR32rr:       I = {4, true, {{CHECK8, X86::OR32ri8},
2631                                            {NOCHECK, X86::OR32ri}}}; break;
2632     case X86::OR64rr:       I = {8, true, {{CHECK8, X86::OR64ri8},
2633                                            {CHECK32, X86::OR64ri32},
2634                                            {NOCHECK, X86::OR64rr}}}; break;
2635 
2636     // XOR
2637     case X86::XOR8rr:       I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
2638     case X86::XOR16rr:      I = {2, true, {{CHECK8, X86::XOR16ri8},
2639                                            {NOCHECK, X86::XOR16ri}}}; break;
2640     case X86::XOR32rr:      I = {4, true, {{CHECK8, X86::XOR32ri8},
2641                                            {NOCHECK, X86::XOR32ri}}}; break;
2642     case X86::XOR64rr:      I = {8, true, {{CHECK8, X86::XOR64ri8},
2643                                            {CHECK32, X86::XOR64ri32},
2644                                            {NOCHECK, X86::XOR64rr}}}; break;
2645     }
2646 
2647     // Compute the new opcode.
2648     unsigned NewOpcode = 0;
2649     for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2650       NewOpcode = Check.second;
2651       if (Check.first == NOCHECK)
2652         break;
2653       if (Check.first == CHECK8 && isInt<8>(Imm))
2654         break;
2655       if (Check.first == CHECK32 && isInt<32>(Imm))
2656         break;
2657     }
2658     if (NewOpcode == Inst.getOpcode())
2659       return false;
2660 
2661     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2662 
2663     unsigned NumFound = 0;
2664     for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2665                   E = InstDesc.getNumOperands();
2666          Index != E; ++Index)
2667       if (Inst.getOperand(Index).isReg() &&
2668           Inst.getOperand(Index).getReg() == Register)
2669         NumFound++;
2670 
2671     if (NumFound != 1)
2672       return false;
2673 
2674     MCOperand TargetOp = Inst.getOperand(0);
2675     Inst.clear();
2676     Inst.setOpcode(NewOpcode);
2677     Inst.addOperand(TargetOp);
2678     if (I.HasLHS)
2679       Inst.addOperand(TargetOp);
2680     Inst.addOperand(MCOperand::createImm(Imm));
2681 
2682     return true;
2683   }
2684 
2685   bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2686                          unsigned ReplaceWith) const override {
2687 
2688     // Get the HasLHS value so that iteration can be done
2689     bool HasLHS;
2690     if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) ||
2691         X86::isSUB(Inst.getOpcode())) {
2692       HasLHS = true;
2693     } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) ||
2694                X86::isTEST(Inst.getOpcode())) {
2695       HasLHS = false;
2696     } else {
2697       switch (Inst.getOpcode()) {
2698       case X86::MOV8rr:
2699       case X86::MOV8rm:
2700       case X86::MOV8mr:
2701       case X86::MOV8ri:
2702       case X86::MOV16rr:
2703       case X86::MOV16rm:
2704       case X86::MOV16mr:
2705       case X86::MOV16ri:
2706       case X86::MOV32rr:
2707       case X86::MOV32rm:
2708       case X86::MOV32mr:
2709       case X86::MOV32ri:
2710       case X86::MOV64rr:
2711       case X86::MOV64rm:
2712       case X86::MOV64mr:
2713       case X86::MOV64ri:
2714       case X86::MOVZX16rr8:
2715       case X86::MOVZX32rr8:
2716       case X86::MOVZX32rr16:
2717       case X86::MOVSX32rm8:
2718       case X86::MOVSX32rr8:
2719       case X86::MOVSX64rm32:
2720       case X86::LEA64r:
2721         HasLHS = false;
2722         break;
2723       default:
2724         return false;
2725       }
2726     }
2727 
2728     const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode());
2729 
2730     bool FoundOne = false;
2731 
2732     // Iterate only through src operands that arent also dest operands
2733     for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2734                   E = InstDesc.getNumOperands();
2735          Index != E; ++Index) {
2736       BitVector RegAliases = getAliases(ToReplace, true);
2737       if (!Inst.getOperand(Index).isReg() ||
2738           !RegAliases.test(Inst.getOperand(Index).getReg()))
2739         continue;
2740       // Resize register if needed
2741       unsigned SizedReplaceWith = getAliasSized(
2742           ReplaceWith, getRegSize(Inst.getOperand(Index).getReg()));
2743       MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith);
2744       Inst.getOperand(Index) = NewOperand;
2745       FoundOne = true;
2746     }
2747 
2748     // Return true if at least one operand was replaced
2749     return FoundOne;
2750   }
2751 
2752   bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2753                           MCContext *Ctx) const override {
2754     Inst.setOpcode(X86::JMP_1);
2755     Inst.addOperand(MCOperand::createExpr(
2756         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
2757     return true;
2758   }
2759 
2760   bool createCall(MCInst &Inst, const MCSymbol *Target,
2761                   MCContext *Ctx) override {
2762     Inst.setOpcode(X86::CALL64pcrel32);
2763     Inst.addOperand(MCOperand::createExpr(
2764         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
2765     return true;
2766   }
2767 
2768   bool createTailCall(MCInst &Inst, const MCSymbol *Target,
2769                       MCContext *Ctx) override {
2770     return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2771   }
2772 
2773   void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2774                           MCContext *Ctx) override {
2775     Seq.clear();
2776     Seq.emplace_back();
2777     createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2778   }
2779 
2780   bool createTrap(MCInst &Inst) const override {
2781     Inst.clear();
2782     Inst.setOpcode(X86::TRAP);
2783     return true;
2784   }
2785 
2786   bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2787                               MCContext *Ctx) const override {
2788     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
2789     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2790     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2791     Inst.getOperand(0) = MCOperand::createExpr(
2792         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2793     return true;
2794   }
2795 
2796   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2797                               unsigned CC) const override {
2798     if (CC == X86::COND_INVALID)
2799       return false;
2800     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC);
2801     Inst.getOperand(0) = MCOperand::createExpr(
2802         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2803     return true;
2804   }
2805 
2806   unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2807     switch (CC) {
2808     default:           return X86::COND_INVALID;
2809 
2810     case X86::COND_E:  return X86::COND_E;
2811     case X86::COND_NE: return X86::COND_E;
2812 
2813     case X86::COND_L:  return X86::COND_L;
2814     case X86::COND_GE: return X86::COND_L;
2815 
2816     case X86::COND_LE: return X86::COND_G;
2817     case X86::COND_G:  return X86::COND_G;
2818 
2819     case X86::COND_B:  return X86::COND_B;
2820     case X86::COND_AE: return X86::COND_B;
2821 
2822     case X86::COND_BE: return X86::COND_A;
2823     case X86::COND_A:  return X86::COND_A;
2824 
2825     case X86::COND_S:  return X86::COND_S;
2826     case X86::COND_NS: return X86::COND_S;
2827 
2828     case X86::COND_P:  return X86::COND_P;
2829     case X86::COND_NP: return X86::COND_P;
2830 
2831     case X86::COND_O:  return X86::COND_O;
2832     case X86::COND_NO: return X86::COND_O;
2833     }
2834   }
2835 
2836   bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
2837                            MCContext *Ctx) const override {
2838     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
2839            "Invalid instruction");
2840     Inst.getOperand(0) = MCOperand::createExpr(
2841         MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
2842     return true;
2843   }
2844 
2845   MCPhysReg getX86R11() const override { return X86::R11; }
2846 
2847   unsigned getShortBranchOpcode(unsigned Opcode) const override {
2848     switch (Opcode) {
2849     default:
2850       return Opcode;
2851     case X86::JMP_2:
2852       return X86::JMP_1;
2853     case X86::JMP_4:
2854       return X86::JMP_1;
2855     case X86::JCC_2:
2856       return X86::JCC_1;
2857     case X86::JCC_4:
2858       return X86::JCC_1;
2859     }
2860   }
2861 
2862   MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
2863     // FIXME: this should depend on the calling convention.
2864     switch (ArgNo) {
2865     case 0:   return X86::RDI;
2866     case 1:   return X86::RSI;
2867     case 2:   return X86::RDX;
2868     case 3:   return X86::RCX;
2869     case 4:   return X86::R8;
2870     case 5:   return X86::R9;
2871     default:  return getNoRegister();
2872     }
2873   }
2874 
2875   void createPause(MCInst &Inst) const override {
2876     Inst.clear();
2877     Inst.setOpcode(X86::PAUSE);
2878   }
2879 
2880   void createLfence(MCInst &Inst) const override {
2881     Inst.clear();
2882     Inst.setOpcode(X86::LFENCE);
2883   }
2884 
2885   bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
2886                         bool IsTailCall) override {
2887     Inst.clear();
2888     Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
2889     Inst.addOperand(MCOperand::createExpr(
2890         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
2891     if (IsTailCall)
2892       setTailCall(Inst);
2893     return true;
2894   }
2895 
2896   void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
2897                       MCContext *Ctx, bool IsTailCall) override {
2898     Seq.clear();
2899     MCInst Inst;
2900     Inst.setOpcode(X86::JMP_1);
2901     Inst.addOperand(MCOperand::createExpr(
2902         MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
2903     if (IsTailCall)
2904       setTailCall(Inst);
2905     Seq.emplace_back(Inst);
2906   }
2907 
2908   bool isConditionalMove(const MCInst &Inst) const override {
2909     unsigned OpCode = Inst.getOpcode();
2910     return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
2911             OpCode == X86::CMOV64rr);
2912   }
2913 
2914   bool isBranchOnMem(const MCInst &Inst) const override {
2915     unsigned OpCode = Inst.getOpcode();
2916     if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
2917         OpCode == X86::JMP64m)
2918       return true;
2919 
2920     return false;
2921   }
2922 
2923   bool isBranchOnReg(const MCInst &Inst) const override {
2924     unsigned OpCode = Inst.getOpcode();
2925     if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
2926         OpCode == X86::JMP64r)
2927       return true;
2928 
2929     return false;
2930   }
2931 
2932   void createPushRegister(MCInst &Inst, MCPhysReg Reg,
2933                           unsigned Size) const override {
2934     Inst.clear();
2935     unsigned NewOpcode = 0;
2936     if (Reg == X86::EFLAGS) {
2937       switch (Size) {
2938       case 2: NewOpcode = X86::PUSHF16;  break;
2939       case 4: NewOpcode = X86::PUSHF32;  break;
2940       case 8: NewOpcode = X86::PUSHF64;  break;
2941       default:
2942         llvm_unreachable("Unexpected size");
2943       }
2944       Inst.setOpcode(NewOpcode);
2945       return;
2946     }
2947     switch (Size) {
2948     case 2: NewOpcode = X86::PUSH16r;  break;
2949     case 4: NewOpcode = X86::PUSH32r;  break;
2950     case 8: NewOpcode = X86::PUSH64r;  break;
2951     default:
2952       llvm_unreachable("Unexpected size");
2953     }
2954     Inst.setOpcode(NewOpcode);
2955     Inst.addOperand(MCOperand::createReg(Reg));
2956   }
2957 
2958   void createPopRegister(MCInst &Inst, MCPhysReg Reg,
2959                          unsigned Size) const override {
2960     Inst.clear();
2961     unsigned NewOpcode = 0;
2962     if (Reg == X86::EFLAGS) {
2963       switch (Size) {
2964       case 2: NewOpcode = X86::POPF16;  break;
2965       case 4: NewOpcode = X86::POPF32;  break;
2966       case 8: NewOpcode = X86::POPF64;  break;
2967       default:
2968         llvm_unreachable("Unexpected size");
2969       }
2970       Inst.setOpcode(NewOpcode);
2971       return;
2972     }
2973     switch (Size) {
2974     case 2: NewOpcode = X86::POP16r;  break;
2975     case 4: NewOpcode = X86::POP32r;  break;
2976     case 8: NewOpcode = X86::POP64r;  break;
2977     default:
2978       llvm_unreachable("Unexpected size");
2979     }
2980     Inst.setOpcode(NewOpcode);
2981     Inst.addOperand(MCOperand::createReg(Reg));
2982   }
2983 
2984   void createPushFlags(MCInst &Inst, unsigned Size) const override {
2985     return createPushRegister(Inst, X86::EFLAGS, Size);
2986   }
2987 
2988   void createPopFlags(MCInst &Inst, unsigned Size) const override {
2989     return createPopRegister(Inst, X86::EFLAGS, Size);
2990   }
2991 
2992   void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
2993                        unsigned Size) const {
2994     unsigned int Opcode;
2995     switch (Size) {
2996     case 1: Opcode = X86::ADD8ri; break;
2997     case 2: Opcode = X86::ADD16ri; break;
2998     case 4: Opcode = X86::ADD32ri; break;
2999     default:
3000       llvm_unreachable("Unexpected size");
3001     }
3002     Inst.setOpcode(Opcode);
3003     Inst.clear();
3004     Inst.addOperand(MCOperand::createReg(Reg));
3005     Inst.addOperand(MCOperand::createReg(Reg));
3006     Inst.addOperand(MCOperand::createImm(Value));
3007   }
3008 
3009   void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3010                                         unsigned Size) const {
3011     unsigned int Opcode;
3012     switch (Size) {
3013     case 1: Opcode = X86::MOV8ri; break;
3014     case 2: Opcode = X86::MOV16ri; break;
3015     case 4: Opcode = X86::MOV32ri; break;
3016     // Writing to a 32-bit register always zeros the upper 32 bits of the
3017     // full-width register
3018     case 8:
3019       Opcode = X86::MOV32ri;
3020       Reg = getAliasSized(Reg, 4);
3021       break;
3022     default:
3023       llvm_unreachable("Unexpected size");
3024     }
3025     Inst.setOpcode(Opcode);
3026     Inst.clear();
3027     Inst.addOperand(MCOperand::createReg(Reg));
3028     Inst.addOperand(MCOperand::createImm(0));
3029   }
3030 
3031   void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3032     Inst.setOpcode(X86::SETCCr);
3033     Inst.clear();
3034     Inst.addOperand(MCOperand::createReg(Reg));
3035     Inst.addOperand(MCOperand::createImm(X86::COND_O));
3036   }
3037 
3038   void createX86Lahf(MCInst &Inst) const {
3039     Inst.setOpcode(X86::LAHF);
3040     Inst.clear();
3041   }
3042 
3043   void createX86Sahf(MCInst &Inst) const {
3044     Inst.setOpcode(X86::SAHF);
3045     Inst.clear();
3046   }
3047 
3048   InstructionListType createInstrIncMemory(const MCSymbol *Target,
3049                                            MCContext *Ctx,
3050                                            bool IsLeaf) const override {
3051     InstructionListType Instrs(IsLeaf ? 13 : 11);
3052     unsigned int I = 0;
3053 
3054     // Don't clobber application red zone (ABI dependent)
3055     if (IsLeaf)
3056       createStackPointerIncrement(Instrs[I++], 128,
3057                                   /*NoFlagsClobber=*/true);
3058 
3059     // Performance improvements based on the optimization discussed at
3060     // https://reviews.llvm.org/D6629
3061     // LAHF/SAHF are used instead of PUSHF/POPF
3062     // PUSHF
3063     createPushRegister(Instrs[I++], X86::RAX, 8);
3064     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3065     createX86Lahf(Instrs[I++]);
3066     createPushRegister(Instrs[I++], X86::RAX, 8);
3067     createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3068     createX86SaveOVFlagToRegister(Instrs[I++], X86::AL);
3069     // LOCK INC
3070     createIncMemory(Instrs[I++], Target, Ctx);
3071     // POPF
3072     createAddRegImm(Instrs[I++], X86::AL, 127, 1);
3073     createPopRegister(Instrs[I++], X86::RAX, 8);
3074     createX86Sahf(Instrs[I++]);
3075     createPopRegister(Instrs[I++], X86::RAX, 8);
3076 
3077     if (IsLeaf)
3078       createStackPointerDecrement(Instrs[I], 128,
3079                                   /*NoFlagsClobber=*/true);
3080     return Instrs;
3081   }
3082 
3083   void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3084                   int64_t Disp) const {
3085     Inst.setOpcode(X86::XCHG64rm);
3086     Inst.addOperand(MCOperand::createReg(Source));
3087     Inst.addOperand(MCOperand::createReg(Source));
3088     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3089     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3090     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3091     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3092     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3093   }
3094 
3095   void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3096                             int64_t Disp) const {
3097     Inst.setOpcode(X86::JMP64m);
3098     Inst.addOperand(MCOperand::createReg(MemBaseReg));      // BaseReg
3099     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3100     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3101     Inst.addOperand(MCOperand::createImm(Disp));            // Displacement
3102     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3103   }
3104 
3105   InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
3106                                                      MCSymbol *HandlerFuncAddr,
3107                                                      int CallSiteID,
3108                                                      MCContext *Ctx) override {
3109     // Check if the target address expression used in the original indirect call
3110     // uses the stack pointer, which we are going to clobber.
3111     static BitVector SPAliases(getAliases(X86::RSP));
3112     bool UsesSP = any_of(useOperands(CallInst), [&](const MCOperand &Op) {
3113       return Op.isReg() && SPAliases[Op.getReg()];
3114     });
3115 
3116     InstructionListType Insts;
3117     MCPhysReg TempReg = getIntArgRegister(0);
3118     // Code sequence used to enter indirect call instrumentation helper:
3119     //   push %rdi
3120     //   add $8, %rsp       ;; $rsp may be used in target, so fix it to prev val
3121     //   movq target, %rdi  ;; via convertIndirectCallTargetToLoad
3122     //   sub $8, %rsp       ;; restore correct stack value
3123     //   push %rdi
3124     //   movq $CallSiteID, %rdi
3125     //   push %rdi
3126     //   callq/jmp HandlerFuncAddr
3127     Insts.emplace_back();
3128     createPushRegister(Insts.back(), TempReg, 8);
3129     if (UsesSP) { // Only adjust SP if we really need to
3130       Insts.emplace_back();
3131       createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3132     }
3133     Insts.emplace_back(CallInst);
3134     // Insts.back() and CallInst now share the same annotation instruction.
3135     // Strip it from Insts.back(), only preserving tail call annotation.
3136     stripAnnotations(Insts.back(), /*KeepTC=*/true);
3137     convertIndirectCallToLoad(Insts.back(), TempReg);
3138     if (UsesSP) {
3139       Insts.emplace_back();
3140       createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false);
3141     }
3142     Insts.emplace_back();
3143     createPushRegister(Insts.back(), TempReg, 8);
3144     Insts.emplace_back();
3145     createLoadImmediate(Insts.back(), TempReg, CallSiteID);
3146     Insts.emplace_back();
3147     createPushRegister(Insts.back(), TempReg, 8);
3148 
3149     MCInst &NewCallInst = Insts.emplace_back();
3150     createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst));
3151 
3152     // Carry over metadata including tail call marker if present.
3153     stripAnnotations(NewCallInst);
3154     moveAnnotations(std::move(CallInst), NewCallInst);
3155 
3156     return Insts;
3157   }
3158 
3159   InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3160     const MCPhysReg TempReg = getIntArgRegister(0);
3161     // We just need to undo the sequence created for every ind call in
3162     // instrumentIndirectTarget(), which can be accomplished minimally with:
3163     //   popfq
3164     //   pop %rdi
3165     //   add $16, %rsp
3166     //   xchg (%rsp), %rdi
3167     //   jmp *-8(%rsp)
3168     InstructionListType Insts(5);
3169     createPopFlags(Insts[0], 8);
3170     createPopRegister(Insts[1], TempReg, 8);
3171     createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false);
3172     createSwap(Insts[3], TempReg, X86::RSP, 0);
3173     createIndirectBranch(Insts[4], X86::RSP, -8);
3174     return Insts;
3175   }
3176 
3177   InstructionListType
3178   createInstrumentedIndTailCallHandlerExitBB() const override {
3179     const MCPhysReg TempReg = getIntArgRegister(0);
3180     // Same thing as above, but for tail calls
3181     //   popfq
3182     //   add $16, %rsp
3183     //   pop %rdi
3184     //   jmp *-16(%rsp)
3185     InstructionListType Insts(4);
3186     createPopFlags(Insts[0], 8);
3187     createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false);
3188     createPopRegister(Insts[2], TempReg, 8);
3189     createIndirectBranch(Insts[3], X86::RSP, -16);
3190     return Insts;
3191   }
3192 
3193   InstructionListType
3194   createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3195                                           const MCSymbol *IndCallHandler,
3196                                           MCContext *Ctx) override {
3197     const MCPhysReg TempReg = getIntArgRegister(0);
3198     // Code sequence used to check whether InstrTampoline was initialized
3199     // and call it if so, returns via IndCallHandler.
3200     //   pushfq
3201     //   mov    InstrTrampoline,%rdi
3202     //   cmp    $0x0,%rdi
3203     //   je     IndCallHandler
3204     //   callq  *%rdi
3205     //   jmpq   IndCallHandler
3206     InstructionListType Insts;
3207     Insts.emplace_back();
3208     createPushFlags(Insts.back(), 8);
3209     Insts.emplace_back();
3210     createMove(Insts.back(), InstrTrampoline, TempReg, Ctx);
3211     InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx);
3212     Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
3213     Insts.emplace_back();
3214     Insts.back().setOpcode(X86::CALL64r);
3215     Insts.back().addOperand(MCOperand::createReg(TempReg));
3216     Insts.emplace_back();
3217     createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true);
3218     return Insts;
3219   }
3220 
3221   InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3222     InstructionListType Insts(2);
3223     MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters");
3224     createMove(Insts[0], NumLocs, X86::EAX, Ctx);
3225     createReturn(Insts[1]);
3226     return Insts;
3227   }
3228 
3229   InstructionListType
3230   createInstrLocationsGetter(MCContext *Ctx) const override {
3231     InstructionListType Insts(2);
3232     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations");
3233     createLea(Insts[0], Locs, X86::EAX, Ctx);
3234     createReturn(Insts[1]);
3235     return Insts;
3236   }
3237 
3238   InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3239     InstructionListType Insts(2);
3240     MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables");
3241     createLea(Insts[0], Locs, X86::EAX, Ctx);
3242     createReturn(Insts[1]);
3243     return Insts;
3244   }
3245 
3246   InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3247     InstructionListType Insts(2);
3248     MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs");
3249     createMove(Insts[0], NumFuncs, X86::EAX, Ctx);
3250     createReturn(Insts[1]);
3251     return Insts;
3252   }
3253 
3254   InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3255                                              MCContext *Ctx) const override {
3256     InstructionListType Insts(1);
3257     createUncondBranch(Insts[0], TgtSym, Ctx);
3258     return Insts;
3259   }
3260 
3261   InstructionListType createDummyReturnFunction(MCContext *Ctx) const override {
3262     InstructionListType Insts(1);
3263     createReturn(Insts[0]);
3264     return Insts;
3265   }
3266 
3267   BlocksVectorTy indirectCallPromotion(
3268       const MCInst &CallInst,
3269       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3270       const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3271       const std::vector<MCInst *> &MethodFetchInsns,
3272       const bool MinimizeCodeSize, MCContext *Ctx) override {
3273     const bool IsTailCall = isTailCall(CallInst);
3274     const bool IsJumpTable = getJumpTable(CallInst) != 0;
3275     BlocksVectorTy Results;
3276 
3277     // Label for the current code block.
3278     MCSymbol *NextTarget = nullptr;
3279 
3280     // The join block which contains all the instructions following CallInst.
3281     // MergeBlock remains null if CallInst is a tail call.
3282     MCSymbol *MergeBlock = nullptr;
3283 
3284     unsigned FuncAddrReg = X86::R10;
3285 
3286     const bool LoadElim = !VtableSyms.empty();
3287     assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3288            "There must be a vtable entry for every method "
3289            "in the targets vector.");
3290 
3291     if (MinimizeCodeSize && !LoadElim) {
3292       std::set<unsigned> UsedRegs;
3293 
3294       for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) {
3295         const MCOperand &Op = CallInst.getOperand(I);
3296         if (Op.isReg())
3297           UsedRegs.insert(Op.getReg());
3298       }
3299 
3300       if (UsedRegs.count(X86::R10) == 0)
3301         FuncAddrReg = X86::R10;
3302       else if (UsedRegs.count(X86::R11) == 0)
3303         FuncAddrReg = X86::R11;
3304       else
3305         return Results;
3306     }
3307 
3308     const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3309       assert(MergeBlock);
3310       NewCall.push_back(CallInst);
3311       MCInst &Merge = NewCall.back();
3312       Merge.clear();
3313       createUncondBranch(Merge, MergeBlock, Ctx);
3314     };
3315 
3316     for (unsigned int i = 0; i < Targets.size(); ++i) {
3317       Results.emplace_back(NextTarget, InstructionListType());
3318       InstructionListType *NewCall = &Results.back().second;
3319 
3320       if (MinimizeCodeSize && !LoadElim) {
3321         // Load the call target into FuncAddrReg.
3322         NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc
3323         MCInst &Target = NewCall->back();
3324         Target.clear();
3325         Target.setOpcode(X86::MOV64ri32);
3326         Target.addOperand(MCOperand::createReg(FuncAddrReg));
3327         if (Targets[i].first) {
3328           // Is this OK?
3329           Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3330               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3331         } else {
3332           const uint64_t Addr = Targets[i].second;
3333           // Immediate address is out of sign extended 32 bit range.
3334           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3335             return BlocksVectorTy();
3336 
3337           Target.addOperand(MCOperand::createImm(Addr));
3338         }
3339 
3340         // Compare current call target to a specific address.
3341         NewCall->push_back(CallInst);
3342         MCInst &Compare = NewCall->back();
3343         Compare.clear();
3344         if (isBranchOnReg(CallInst))
3345           Compare.setOpcode(X86::CMP64rr);
3346         else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3347           Compare.setOpcode(X86::CMP64ri32);
3348         else
3349           Compare.setOpcode(X86::CMP64rm);
3350 
3351         Compare.addOperand(MCOperand::createReg(FuncAddrReg));
3352 
3353         // TODO: Would be preferable to only load this value once.
3354         for (unsigned i = 0;
3355              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3356           if (!CallInst.getOperand(i).isInst())
3357             Compare.addOperand(CallInst.getOperand(i));
3358       } else {
3359         // Compare current call target to a specific address.
3360         NewCall->push_back(CallInst);
3361         MCInst &Compare = NewCall->back();
3362         Compare.clear();
3363         if (isBranchOnReg(CallInst))
3364           Compare.setOpcode(X86::CMP64ri32);
3365         else
3366           Compare.setOpcode(X86::CMP64mi32);
3367 
3368         // Original call address.
3369         for (unsigned i = 0;
3370              i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i)
3371           if (!CallInst.getOperand(i).isInst())
3372             Compare.addOperand(CallInst.getOperand(i));
3373 
3374         // Target address.
3375         if (Targets[i].first || LoadElim) {
3376           const MCSymbol *Sym =
3377               LoadElim ? VtableSyms[i].first : Targets[i].first;
3378           const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3379           const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx);
3380           if (Addend)
3381             Expr = MCBinaryExpr::createAdd(
3382                 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx);
3383           Compare.addOperand(MCOperand::createExpr(Expr));
3384         } else {
3385           const uint64_t Addr = Targets[i].second;
3386           // Immediate address is out of sign extended 32 bit range.
3387           if (int64_t(Addr) != int64_t(int32_t(Addr)))
3388             return BlocksVectorTy();
3389 
3390           Compare.addOperand(MCOperand::createImm(Addr));
3391         }
3392       }
3393 
3394       // jump to next target compare.
3395       NextTarget =
3396           Ctx->createNamedTempSymbol(); // generate label for the next block
3397       NewCall->push_back(CallInst);
3398 
3399       if (IsJumpTable) {
3400         MCInst &Je = NewCall->back();
3401 
3402         // Jump to next compare if target addresses don't match.
3403         Je.clear();
3404         Je.setOpcode(X86::JCC_1);
3405         if (Targets[i].first)
3406           Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3407               Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3408         else
3409           Je.addOperand(MCOperand::createImm(Targets[i].second));
3410 
3411         Je.addOperand(MCOperand::createImm(X86::COND_E));
3412         assert(!isInvoke(CallInst));
3413       } else {
3414         MCInst &Jne = NewCall->back();
3415 
3416         // Jump to next compare if target addresses don't match.
3417         Jne.clear();
3418         Jne.setOpcode(X86::JCC_1);
3419         Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3420             NextTarget, MCSymbolRefExpr::VK_None, *Ctx)));
3421         Jne.addOperand(MCOperand::createImm(X86::COND_NE));
3422 
3423         // Call specific target directly.
3424         Results.emplace_back(Ctx->createNamedTempSymbol(),
3425                              InstructionListType());
3426         NewCall = &Results.back().second;
3427         NewCall->push_back(CallInst);
3428         MCInst &CallOrJmp = NewCall->back();
3429 
3430         CallOrJmp.clear();
3431 
3432         if (MinimizeCodeSize && !LoadElim) {
3433           CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3434           CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg));
3435         } else {
3436           CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3437 
3438           if (Targets[i].first)
3439             CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3440                 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3441           else
3442             CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second));
3443         }
3444         if (IsTailCall)
3445           setTailCall(CallOrJmp);
3446 
3447         if (CallOrJmp.getOpcode() == X86::CALL64r ||
3448             CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3449           if (std::optional<uint32_t> Offset = getOffset(CallInst))
3450             // Annotated as duplicated call
3451             setOffset(CallOrJmp, *Offset);
3452         }
3453 
3454         if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) {
3455           // Copy over any EH or GNU args size information from the original
3456           // call.
3457           std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
3458           if (EHInfo)
3459             addEHInfo(CallOrJmp, *EHInfo);
3460           int64_t GnuArgsSize = getGnuArgsSize(CallInst);
3461           if (GnuArgsSize >= 0)
3462             addGnuArgsSize(CallOrJmp, GnuArgsSize);
3463         }
3464 
3465         if (!IsTailCall) {
3466           // The fallthrough block for the most common target should be
3467           // the merge block.
3468           if (i == 0) {
3469             // Fallthrough to merge block.
3470             MergeBlock = Ctx->createNamedTempSymbol();
3471           } else {
3472             // Insert jump to the merge block if we are not doing a fallthrough.
3473             jumpToMergeBlock(*NewCall);
3474           }
3475         }
3476       }
3477     }
3478 
3479     // Cold call block.
3480     Results.emplace_back(NextTarget, InstructionListType());
3481     InstructionListType &NewCall = Results.back().second;
3482     for (const MCInst *Inst : MethodFetchInsns)
3483       if (Inst != &CallInst)
3484         NewCall.push_back(*Inst);
3485     NewCall.push_back(CallInst);
3486 
3487     // Jump to merge block from cold call block
3488     if (!IsTailCall && !IsJumpTable) {
3489       jumpToMergeBlock(NewCall);
3490 
3491       // Record merge block
3492       Results.emplace_back(MergeBlock, InstructionListType());
3493     }
3494 
3495     return Results;
3496   }
3497 
3498   BlocksVectorTy jumpTablePromotion(
3499       const MCInst &IJmpInst,
3500       const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3501       const std::vector<MCInst *> &TargetFetchInsns,
3502       MCContext *Ctx) const override {
3503     assert(getJumpTable(IJmpInst) != 0);
3504     uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg");
3505     if (IndexReg == 0)
3506       return BlocksVectorTy();
3507 
3508     BlocksVectorTy Results;
3509 
3510     // Label for the current code block.
3511     MCSymbol *NextTarget = nullptr;
3512 
3513     for (unsigned int i = 0; i < Targets.size(); ++i) {
3514       Results.emplace_back(NextTarget, InstructionListType());
3515       InstructionListType *CurBB = &Results.back().second;
3516 
3517       // Compare current index to a specific index.
3518       CurBB->emplace_back(MCInst());
3519       MCInst &CompareInst = CurBB->back();
3520       CompareInst.setLoc(IJmpInst.getLoc());
3521       CompareInst.setOpcode(X86::CMP64ri32);
3522       CompareInst.addOperand(MCOperand::createReg(IndexReg));
3523 
3524       const uint64_t CaseIdx = Targets[i].second;
3525       // Immediate address is out of sign extended 32 bit range.
3526       if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3527         return BlocksVectorTy();
3528 
3529       CompareInst.addOperand(MCOperand::createImm(CaseIdx));
3530       shortenInstruction(CompareInst, *Ctx->getSubtargetInfo());
3531 
3532       // jump to next target compare.
3533       NextTarget =
3534           Ctx->createNamedTempSymbol(); // generate label for the next block
3535       CurBB->push_back(MCInst());
3536 
3537       MCInst &JEInst = CurBB->back();
3538       JEInst.setLoc(IJmpInst.getLoc());
3539 
3540       // Jump to target if indices match
3541       JEInst.setOpcode(X86::JCC_1);
3542       JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
3543           Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3544       JEInst.addOperand(MCOperand::createImm(X86::COND_E));
3545     }
3546 
3547     // Cold call block.
3548     Results.emplace_back(NextTarget, InstructionListType());
3549     InstructionListType &CurBB = Results.back().second;
3550     for (const MCInst *Inst : TargetFetchInsns)
3551       if (Inst != &IJmpInst)
3552         CurBB.push_back(*Inst);
3553 
3554     CurBB.push_back(IJmpInst);
3555 
3556     return Results;
3557   }
3558 
3559 private:
3560   bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3561                   MCContext *Ctx) const {
3562     Inst.setOpcode(X86::MOV64rm);
3563     Inst.addOperand(MCOperand::createReg(Reg));
3564     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3565     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3566     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3567     Inst.addOperand(MCOperand::createExpr(
3568         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3569                                 *Ctx)));                    // Displacement
3570     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3571 
3572     return true;
3573   }
3574 
3575   bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3576                  MCContext *Ctx) const {
3577     Inst.setOpcode(X86::LEA64r);
3578     Inst.addOperand(MCOperand::createReg(Reg));
3579     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
3580     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
3581     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3582     Inst.addOperand(MCOperand::createExpr(
3583         MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
3584                                 *Ctx)));                    // Displacement
3585     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3586     return true;
3587   }
3588 };
3589 
3590 } // namespace
3591 
3592 namespace llvm {
3593 namespace bolt {
3594 
3595 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3596                                       const MCInstrInfo *Info,
3597                                       const MCRegisterInfo *RegInfo) {
3598   return new X86MCPlusBuilder(Analysis, Info, RegInfo);
3599 }
3600 
3601 } // namespace bolt
3602 } // namespace llvm
3603