1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides X86-specific MCPlus builder. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86InstrRelaxTables.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "X86MCSymbolizer.h" 17 #include "bolt/Core/MCPlus.h" 18 #include "bolt/Core/MCPlusBuilder.h" 19 #include "llvm/BinaryFormat/ELF.h" 20 #include "llvm/MC/MCContext.h" 21 #include "llvm/MC/MCFixupKindInfo.h" 22 #include "llvm/MC/MCInst.h" 23 #include "llvm/MC/MCInstBuilder.h" 24 #include "llvm/MC/MCInstrInfo.h" 25 #include "llvm/MC/MCRegister.h" 26 #include "llvm/MC/MCRegisterInfo.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/DataExtractor.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/Errc.h" 31 #include "llvm/Support/ErrorHandling.h" 32 #include "llvm/Support/ErrorOr.h" 33 #include <set> 34 35 #define DEBUG_TYPE "mcplus" 36 37 using namespace llvm; 38 using namespace bolt; 39 40 namespace opts { 41 42 extern cl::OptionCategory BoltOptCategory; 43 44 static cl::opt<bool> X86StripRedundantAddressSize( 45 "x86-strip-redundant-address-size", 46 cl::desc("Remove redundant Address-Size override prefix"), cl::init(true), 47 cl::cat(BoltOptCategory)); 48 49 } // namespace opts 50 51 namespace { 52 53 unsigned getShortArithOpcode(unsigned Opcode) { 54 return X86::getShortOpcodeArith(Opcode); 55 } 56 57 bool isMOVSX64rm32(const MCInst &Inst) { 58 return Inst.getOpcode() == X86::MOVSX64rm32; 59 } 60 61 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; } 62 63 bool isADDri(const MCInst &Inst) { 64 return Inst.getOpcode() == X86::ADD64ri32 || 65 Inst.getOpcode() == X86::ADD64ri8; 66 } 67 68 #define GET_INSTRINFO_OPERAND_TYPES_ENUM 69 #define GET_INSTRINFO_OPERAND_TYPE 70 #define GET_INSTRINFO_MEM_OPERAND_SIZE 71 #include "X86GenInstrInfo.inc" 72 73 class X86MCPlusBuilder : public MCPlusBuilder { 74 public: 75 X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, 76 const MCRegisterInfo *RegInfo) 77 : MCPlusBuilder(Analysis, Info, RegInfo) {} 78 79 std::unique_ptr<MCSymbolizer> 80 createTargetSymbolizer(BinaryFunction &Function) const override { 81 return std::make_unique<X86MCSymbolizer>(Function); 82 } 83 84 bool isBranch(const MCInst &Inst) const override { 85 return Analysis->isBranch(Inst) && !isTailCall(Inst); 86 } 87 88 bool isNoop(const MCInst &Inst) const override { 89 return X86::isNOP(Inst.getOpcode()); 90 } 91 92 unsigned getCondCode(const MCInst &Inst) const override { 93 unsigned Opcode = Inst.getOpcode(); 94 if (X86::isJCC(Opcode)) 95 return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm(); 96 return X86::COND_INVALID; 97 } 98 99 unsigned getInvertedCondCode(unsigned CC) const override { 100 switch (CC) { 101 default: return X86::COND_INVALID; 102 case X86::COND_E: return X86::COND_NE; 103 case X86::COND_NE: return X86::COND_E; 104 case X86::COND_L: return X86::COND_GE; 105 case X86::COND_LE: return X86::COND_G; 106 case X86::COND_G: return X86::COND_LE; 107 case X86::COND_GE: return X86::COND_L; 108 case X86::COND_B: return X86::COND_AE; 109 case X86::COND_BE: return X86::COND_A; 110 case X86::COND_A: return X86::COND_BE; 111 case X86::COND_AE: return X86::COND_B; 112 case X86::COND_S: return X86::COND_NS; 113 case X86::COND_NS: return X86::COND_S; 114 case X86::COND_P: return X86::COND_NP; 115 case X86::COND_NP: return X86::COND_P; 116 case X86::COND_O: return X86::COND_NO; 117 case X86::COND_NO: return X86::COND_O; 118 } 119 } 120 121 unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override { 122 enum DecodedCondCode : uint8_t { 123 DCC_EQUAL = 0x1, 124 DCC_GREATER = 0x2, 125 DCC_LESSER = 0x4, 126 DCC_GREATER_OR_LESSER = 0x6, 127 DCC_UNSIGNED = 0x8, 128 DCC_SIGNED = 0x10, 129 DCC_INVALID = 0x20, 130 }; 131 132 auto decodeCondCode = [&](unsigned CC) -> uint8_t { 133 switch (CC) { 134 default: return DCC_INVALID; 135 case X86::COND_E: return DCC_EQUAL; 136 case X86::COND_NE: return DCC_GREATER | DCC_LESSER; 137 case X86::COND_L: return DCC_LESSER | DCC_SIGNED; 138 case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED; 139 case X86::COND_G: return DCC_GREATER | DCC_SIGNED; 140 case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED; 141 case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED; 142 case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED; 143 case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED; 144 case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED; 145 } 146 }; 147 148 uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2); 149 150 if (DCC & DCC_INVALID) 151 return X86::COND_INVALID; 152 153 if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED) 154 return X86::COND_INVALID; 155 156 switch (DCC) { 157 default: return X86::COND_INVALID; 158 case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE; 159 case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE; 160 case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE; 161 case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE; 162 case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE; 163 case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE; 164 case DCC_GREATER | DCC_LESSER: return X86::COND_NE; 165 case DCC_EQUAL | DCC_SIGNED: return X86::COND_E; 166 case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E; 167 case DCC_EQUAL: return X86::COND_E; 168 case DCC_LESSER | DCC_SIGNED: return X86::COND_L; 169 case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B; 170 case DCC_GREATER | DCC_SIGNED: return X86::COND_G; 171 case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A; 172 } 173 } 174 175 bool isValidCondCode(unsigned CC) const override { 176 return (CC != X86::COND_INVALID); 177 } 178 179 bool isBreakpoint(const MCInst &Inst) const override { 180 return Inst.getOpcode() == X86::INT3; 181 } 182 183 bool isPrefix(const MCInst &Inst) const override { 184 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 185 return X86II::isPrefix(Desc.TSFlags); 186 } 187 188 bool isRep(const MCInst &Inst) const override { 189 return Inst.getFlags() == X86::IP_HAS_REPEAT; 190 } 191 192 bool deleteREPPrefix(MCInst &Inst) const override { 193 if (Inst.getFlags() == X86::IP_HAS_REPEAT) { 194 Inst.setFlags(0); 195 return true; 196 } 197 return false; 198 } 199 200 // FIXME: For compatibility with old LLVM only! 201 bool isTerminator(const MCInst &Inst) const override { 202 unsigned Opcode = Inst.getOpcode(); 203 return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) || 204 X86::isUD2(Opcode); 205 } 206 207 bool isIndirectCall(const MCInst &Inst) const override { 208 return isCall(Inst) && 209 ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); 210 } 211 212 bool isPop(const MCInst &Inst) const override { 213 return getPopSize(Inst) == 0 ? false : true; 214 } 215 216 bool isTerminateBranch(const MCInst &Inst) const override { 217 return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64; 218 } 219 220 int getPopSize(const MCInst &Inst) const override { 221 switch (Inst.getOpcode()) { 222 case X86::POP16r: 223 case X86::POP16rmm: 224 case X86::POP16rmr: 225 case X86::POPF16: 226 case X86::POPA16: 227 case X86::POPDS16: 228 case X86::POPES16: 229 case X86::POPFS16: 230 case X86::POPGS16: 231 case X86::POPSS16: 232 return 2; 233 case X86::POP32r: 234 case X86::POP32rmm: 235 case X86::POP32rmr: 236 case X86::POPA32: 237 case X86::POPDS32: 238 case X86::POPES32: 239 case X86::POPF32: 240 case X86::POPFS32: 241 case X86::POPGS32: 242 case X86::POPSS32: 243 return 4; 244 case X86::POP64r: 245 case X86::POP64rmm: 246 case X86::POP64rmr: 247 case X86::POPF64: 248 case X86::POPFS64: 249 case X86::POPGS64: 250 return 8; 251 } 252 return 0; 253 } 254 255 bool isPush(const MCInst &Inst) const override { 256 return getPushSize(Inst) == 0 ? false : true; 257 } 258 259 int getPushSize(const MCInst &Inst) const override { 260 switch (Inst.getOpcode()) { 261 case X86::PUSH16i8: 262 case X86::PUSH16r: 263 case X86::PUSH16rmm: 264 case X86::PUSH16rmr: 265 case X86::PUSHA16: 266 case X86::PUSHCS16: 267 case X86::PUSHDS16: 268 case X86::PUSHES16: 269 case X86::PUSHF16: 270 case X86::PUSHFS16: 271 case X86::PUSHGS16: 272 case X86::PUSHSS16: 273 case X86::PUSH16i: 274 return 2; 275 case X86::PUSH32i8: 276 case X86::PUSH32r: 277 case X86::PUSH32rmm: 278 case X86::PUSH32rmr: 279 case X86::PUSHA32: 280 case X86::PUSHCS32: 281 case X86::PUSHDS32: 282 case X86::PUSHES32: 283 case X86::PUSHF32: 284 case X86::PUSHFS32: 285 case X86::PUSHGS32: 286 case X86::PUSHSS32: 287 case X86::PUSH32i: 288 return 4; 289 case X86::PUSH64i32: 290 case X86::PUSH64i8: 291 case X86::PUSH64r: 292 case X86::PUSH64rmm: 293 case X86::PUSH64rmr: 294 case X86::PUSHF64: 295 case X86::PUSHFS64: 296 case X86::PUSHGS64: 297 return 8; 298 } 299 return 0; 300 } 301 302 bool isSUB(const MCInst &Inst) const override { 303 return X86::isSUB(Inst.getOpcode()); 304 } 305 306 bool isLEA64r(const MCInst &Inst) const override { 307 return Inst.getOpcode() == X86::LEA64r; 308 } 309 310 bool isLeave(const MCInst &Inst) const override { 311 return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64; 312 } 313 314 bool isMoveMem2Reg(const MCInst &Inst) const override { 315 switch (Inst.getOpcode()) { 316 case X86::MOV16rm: 317 case X86::MOV32rm: 318 case X86::MOV64rm: 319 return true; 320 } 321 return false; 322 } 323 324 bool isUnsupportedBranch(unsigned Opcode) const override { 325 switch (Opcode) { 326 default: 327 return false; 328 case X86::LOOP: 329 case X86::LOOPE: 330 case X86::LOOPNE: 331 case X86::JECXZ: 332 case X86::JRCXZ: 333 return true; 334 } 335 } 336 337 bool isLoad(const MCInst &Inst) const override { 338 if (isPop(Inst)) 339 return true; 340 341 int MemOpNo = getMemoryOperandNo(Inst); 342 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 343 344 if (MemOpNo == -1) 345 return false; 346 347 return MCII.mayLoad(); 348 } 349 350 bool isStore(const MCInst &Inst) const override { 351 if (isPush(Inst)) 352 return true; 353 354 int MemOpNo = getMemoryOperandNo(Inst); 355 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 356 357 if (MemOpNo == -1) 358 return false; 359 360 return MCII.mayStore(); 361 } 362 363 bool isCleanRegXOR(const MCInst &Inst) const override { 364 switch (Inst.getOpcode()) { 365 case X86::XOR16rr: 366 case X86::XOR32rr: 367 case X86::XOR64rr: 368 break; 369 default: 370 return false; 371 } 372 return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg()); 373 } 374 375 bool isPacked(const MCInst &Inst) const override { 376 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 377 return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD; 378 } 379 380 bool shouldRecordCodeRelocation(uint64_t RelType) const override { 381 switch (RelType) { 382 case ELF::R_X86_64_8: 383 case ELF::R_X86_64_16: 384 case ELF::R_X86_64_32: 385 case ELF::R_X86_64_32S: 386 case ELF::R_X86_64_64: 387 case ELF::R_X86_64_PC8: 388 case ELF::R_X86_64_PC32: 389 case ELF::R_X86_64_PC64: 390 case ELF::R_X86_64_GOTPCRELX: 391 case ELF::R_X86_64_REX_GOTPCRELX: 392 return true; 393 case ELF::R_X86_64_PLT32: 394 case ELF::R_X86_64_GOTPCREL: 395 case ELF::R_X86_64_TPOFF32: 396 case ELF::R_X86_64_GOTTPOFF: 397 return false; 398 default: 399 llvm_unreachable("Unexpected x86 relocation type in code"); 400 } 401 } 402 403 unsigned getTrapFillValue() const override { return 0xCC; } 404 405 struct IndJmpMatcherFrag1 : MCInstMatcher { 406 std::unique_ptr<MCInstMatcher> Base; 407 std::unique_ptr<MCInstMatcher> Scale; 408 std::unique_ptr<MCInstMatcher> Index; 409 std::unique_ptr<MCInstMatcher> Offset; 410 411 IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 412 std::unique_ptr<MCInstMatcher> Scale, 413 std::unique_ptr<MCInstMatcher> Index, 414 std::unique_ptr<MCInstMatcher> Offset) 415 : Base(std::move(Base)), Scale(std::move(Scale)), 416 Index(std::move(Index)), Offset(std::move(Offset)) {} 417 418 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 419 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 420 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 421 return false; 422 423 if (CurInst->getOpcode() != X86::JMP64m) 424 return false; 425 426 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 427 if (MemOpNo == -1) 428 return false; 429 430 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 431 return false; 432 if (!Scale->match(MRI, MIB, this->InstrWindow, 433 MemOpNo + X86::AddrScaleAmt)) 434 return false; 435 if (!Index->match(MRI, MIB, this->InstrWindow, 436 MemOpNo + X86::AddrIndexReg)) 437 return false; 438 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 439 return false; 440 return true; 441 } 442 443 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 444 MIB.addAnnotation(*CurInst, Annotation, true); 445 Base->annotate(MIB, Annotation); 446 Scale->annotate(MIB, Annotation); 447 Index->annotate(MIB, Annotation); 448 Offset->annotate(MIB, Annotation); 449 } 450 }; 451 452 std::unique_ptr<MCInstMatcher> 453 matchIndJmp(std::unique_ptr<MCInstMatcher> Base, 454 std::unique_ptr<MCInstMatcher> Scale, 455 std::unique_ptr<MCInstMatcher> Index, 456 std::unique_ptr<MCInstMatcher> Offset) const override { 457 return std::unique_ptr<MCInstMatcher>( 458 new IndJmpMatcherFrag1(std::move(Base), std::move(Scale), 459 std::move(Index), std::move(Offset))); 460 } 461 462 struct IndJmpMatcherFrag2 : MCInstMatcher { 463 std::unique_ptr<MCInstMatcher> Reg; 464 465 IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg) 466 : Reg(std::move(Reg)) {} 467 468 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 469 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 470 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 471 return false; 472 473 if (CurInst->getOpcode() != X86::JMP64r) 474 return false; 475 476 return Reg->match(MRI, MIB, this->InstrWindow, 0); 477 } 478 479 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 480 MIB.addAnnotation(*CurInst, Annotation, true); 481 Reg->annotate(MIB, Annotation); 482 } 483 }; 484 485 std::unique_ptr<MCInstMatcher> 486 matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override { 487 return std::unique_ptr<MCInstMatcher>( 488 new IndJmpMatcherFrag2(std::move(Target))); 489 } 490 491 struct LoadMatcherFrag1 : MCInstMatcher { 492 std::unique_ptr<MCInstMatcher> Base; 493 std::unique_ptr<MCInstMatcher> Scale; 494 std::unique_ptr<MCInstMatcher> Index; 495 std::unique_ptr<MCInstMatcher> Offset; 496 497 LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 498 std::unique_ptr<MCInstMatcher> Scale, 499 std::unique_ptr<MCInstMatcher> Index, 500 std::unique_ptr<MCInstMatcher> Offset) 501 : Base(std::move(Base)), Scale(std::move(Scale)), 502 Index(std::move(Index)), Offset(std::move(Offset)) {} 503 504 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 505 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 506 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 507 return false; 508 509 if (CurInst->getOpcode() != X86::MOV64rm && 510 CurInst->getOpcode() != X86::MOVSX64rm32) 511 return false; 512 513 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 514 if (MemOpNo == -1) 515 return false; 516 517 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 518 return false; 519 if (!Scale->match(MRI, MIB, this->InstrWindow, 520 MemOpNo + X86::AddrScaleAmt)) 521 return false; 522 if (!Index->match(MRI, MIB, this->InstrWindow, 523 MemOpNo + X86::AddrIndexReg)) 524 return false; 525 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 526 return false; 527 return true; 528 } 529 530 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 531 MIB.addAnnotation(*CurInst, Annotation, true); 532 Base->annotate(MIB, Annotation); 533 Scale->annotate(MIB, Annotation); 534 Index->annotate(MIB, Annotation); 535 Offset->annotate(MIB, Annotation); 536 } 537 }; 538 539 std::unique_ptr<MCInstMatcher> 540 matchLoad(std::unique_ptr<MCInstMatcher> Base, 541 std::unique_ptr<MCInstMatcher> Scale, 542 std::unique_ptr<MCInstMatcher> Index, 543 std::unique_ptr<MCInstMatcher> Offset) const override { 544 return std::unique_ptr<MCInstMatcher>( 545 new LoadMatcherFrag1(std::move(Base), std::move(Scale), 546 std::move(Index), std::move(Offset))); 547 } 548 549 struct AddMatcher : MCInstMatcher { 550 std::unique_ptr<MCInstMatcher> A; 551 std::unique_ptr<MCInstMatcher> B; 552 553 AddMatcher(std::unique_ptr<MCInstMatcher> A, 554 std::unique_ptr<MCInstMatcher> B) 555 : A(std::move(A)), B(std::move(B)) {} 556 557 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 558 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 559 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 560 return false; 561 562 if (CurInst->getOpcode() == X86::ADD64rr || 563 CurInst->getOpcode() == X86::ADD64rr_DB || 564 CurInst->getOpcode() == X86::ADD64rr_REV) { 565 if (!A->match(MRI, MIB, this->InstrWindow, 1)) { 566 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 567 return false; 568 return A->match(MRI, MIB, this->InstrWindow, 2); 569 } 570 571 if (B->match(MRI, MIB, this->InstrWindow, 2)) 572 return true; 573 574 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 575 return false; 576 return A->match(MRI, MIB, this->InstrWindow, 2); 577 } 578 579 return false; 580 } 581 582 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 583 MIB.addAnnotation(*CurInst, Annotation, true); 584 A->annotate(MIB, Annotation); 585 B->annotate(MIB, Annotation); 586 } 587 }; 588 589 std::unique_ptr<MCInstMatcher> 590 matchAdd(std::unique_ptr<MCInstMatcher> A, 591 std::unique_ptr<MCInstMatcher> B) const override { 592 return std::unique_ptr<MCInstMatcher>( 593 new AddMatcher(std::move(A), std::move(B))); 594 } 595 596 struct LEAMatcher : MCInstMatcher { 597 std::unique_ptr<MCInstMatcher> Target; 598 599 LEAMatcher(std::unique_ptr<MCInstMatcher> Target) 600 : Target(std::move(Target)) {} 601 602 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 603 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 604 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 605 return false; 606 607 if (CurInst->getOpcode() != X86::LEA64r) 608 return false; 609 610 if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 || 611 CurInst->getOperand(1 + X86::AddrIndexReg).getReg() != 612 X86::NoRegister || 613 (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != 614 X86::NoRegister && 615 CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP)) 616 return false; 617 618 return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp); 619 } 620 621 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 622 MIB.addAnnotation(*CurInst, Annotation, true); 623 Target->annotate(MIB, Annotation); 624 } 625 }; 626 627 std::unique_ptr<MCInstMatcher> 628 matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override { 629 return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target))); 630 } 631 632 bool hasPCRelOperand(const MCInst &Inst) const override { 633 for (const MCOperand &Operand : Inst) 634 if (Operand.isReg() && Operand.getReg() == X86::RIP) 635 return true; 636 return false; 637 } 638 639 int getMemoryOperandNo(const MCInst &Inst) const override { 640 unsigned Opcode = Inst.getOpcode(); 641 const MCInstrDesc &Desc = Info->get(Opcode); 642 int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags); 643 if (MemOpNo >= 0) 644 MemOpNo += X86II::getOperandBias(Desc); 645 return MemOpNo; 646 } 647 648 bool hasEVEXEncoding(const MCInst &Inst) const override { 649 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 650 return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX; 651 } 652 653 bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override { 654 const auto *I = Insts.begin(); 655 while (I != Insts.end() && isPrefix(*I)) 656 ++I; 657 if (I == Insts.end()) 658 return false; 659 660 const MCInst &FirstInst = *I; 661 ++I; 662 while (I != Insts.end() && isPrefix(*I)) 663 ++I; 664 if (I == Insts.end()) 665 return false; 666 const MCInst &SecondInst = *I; 667 668 if (!isConditionalBranch(SecondInst)) 669 return false; 670 // Cannot fuse if the first instruction uses RIP-relative memory. 671 if (hasPCRelOperand(FirstInst)) 672 return false; 673 674 const X86::FirstMacroFusionInstKind CmpKind = 675 X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode()); 676 if (CmpKind == X86::FirstMacroFusionInstKind::Invalid) 677 return false; 678 679 X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst)); 680 X86::SecondMacroFusionInstKind BranchKind = 681 X86::classifySecondCondCodeInMacroFusion(CC); 682 if (BranchKind == X86::SecondMacroFusionInstKind::Invalid) 683 return false; 684 return X86::isMacroFused(CmpKind, BranchKind); 685 } 686 687 std::optional<X86MemOperand> 688 evaluateX86MemoryOperand(const MCInst &Inst) const override { 689 int MemOpNo = getMemoryOperandNo(Inst); 690 if (MemOpNo < 0) 691 return std::nullopt; 692 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 693 694 if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst)) 695 return std::nullopt; 696 697 const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg); 698 const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt); 699 const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg); 700 const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp); 701 const MCOperand &Segment = 702 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg); 703 704 // Make sure it is a well-formed memory operand. 705 if (!Base.isReg() || !Scale.isImm() || !Index.isReg() || 706 (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg()) 707 return std::nullopt; 708 709 X86MemOperand MO; 710 MO.BaseRegNum = Base.getReg(); 711 MO.ScaleImm = Scale.getImm(); 712 MO.IndexRegNum = Index.getReg(); 713 MO.DispImm = Disp.isImm() ? Disp.getImm() : 0; 714 MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr; 715 MO.SegRegNum = Segment.getReg(); 716 return MO; 717 } 718 719 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, 720 uint64_t Address, 721 uint64_t Size) const override { 722 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 723 if (!MO) 724 return false; 725 726 // Make sure it's a well-formed addressing we can statically evaluate. 727 if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) || 728 MO->IndexRegNum != X86::NoRegister || 729 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 730 return false; 731 732 Target = MO->DispImm; 733 if (MO->BaseRegNum == X86::RIP) { 734 assert(Size != 0 && "instruction size required in order to statically " 735 "evaluate RIP-relative address"); 736 Target += Address + Size; 737 } 738 return true; 739 } 740 741 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { 742 int MemOpNo = getMemoryOperandNo(Inst); 743 if (MemOpNo < 0) 744 return Inst.end(); 745 return Inst.begin() + (MemOpNo + X86::AddrDisp); 746 } 747 748 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { 749 MCOperand *OI = getMemOperandDisp(Inst); 750 if (OI == Inst.end()) 751 return false; 752 *OI = Operand; 753 return true; 754 } 755 756 /// Get the registers used as function parameters. 757 /// This function is specific to the x86_64 abi on Linux. 758 BitVector getRegsUsedAsParams() const override { 759 BitVector Regs = BitVector(RegInfo->getNumRegs(), false); 760 Regs |= getAliases(X86::RSI); 761 Regs |= getAliases(X86::RDI); 762 Regs |= getAliases(X86::RDX); 763 Regs |= getAliases(X86::RCX); 764 Regs |= getAliases(X86::R8); 765 Regs |= getAliases(X86::R9); 766 return Regs; 767 } 768 769 void getCalleeSavedRegs(BitVector &Regs) const override { 770 Regs |= getAliases(X86::RBX); 771 Regs |= getAliases(X86::RBP); 772 Regs |= getAliases(X86::R12); 773 Regs |= getAliases(X86::R13); 774 Regs |= getAliases(X86::R14); 775 Regs |= getAliases(X86::R15); 776 } 777 778 void getDefaultDefIn(BitVector &Regs) const override { 779 assert(Regs.size() >= RegInfo->getNumRegs() && 780 "The size of BitVector is less than RegInfo->getNumRegs()."); 781 Regs.set(X86::RAX); 782 Regs.set(X86::RCX); 783 Regs.set(X86::RDX); 784 Regs.set(X86::RSI); 785 Regs.set(X86::RDI); 786 Regs.set(X86::R8); 787 Regs.set(X86::R9); 788 Regs.set(X86::XMM0); 789 Regs.set(X86::XMM1); 790 Regs.set(X86::XMM2); 791 Regs.set(X86::XMM3); 792 Regs.set(X86::XMM4); 793 Regs.set(X86::XMM5); 794 Regs.set(X86::XMM6); 795 Regs.set(X86::XMM7); 796 } 797 798 void getDefaultLiveOut(BitVector &Regs) const override { 799 assert(Regs.size() >= RegInfo->getNumRegs() && 800 "The size of BitVector is less than RegInfo->getNumRegs()."); 801 Regs |= getAliases(X86::RAX); 802 Regs |= getAliases(X86::RDX); 803 Regs |= getAliases(X86::RCX); 804 Regs |= getAliases(X86::XMM0); 805 Regs |= getAliases(X86::XMM1); 806 } 807 808 void getGPRegs(BitVector &Regs, bool IncludeAlias) const override { 809 if (IncludeAlias) { 810 Regs |= getAliases(X86::RAX); 811 Regs |= getAliases(X86::RBX); 812 Regs |= getAliases(X86::RBP); 813 Regs |= getAliases(X86::RSI); 814 Regs |= getAliases(X86::RDI); 815 Regs |= getAliases(X86::RDX); 816 Regs |= getAliases(X86::RCX); 817 Regs |= getAliases(X86::R8); 818 Regs |= getAliases(X86::R9); 819 Regs |= getAliases(X86::R10); 820 Regs |= getAliases(X86::R11); 821 Regs |= getAliases(X86::R12); 822 Regs |= getAliases(X86::R13); 823 Regs |= getAliases(X86::R14); 824 Regs |= getAliases(X86::R15); 825 return; 826 } 827 Regs.set(X86::RAX); 828 Regs.set(X86::RBX); 829 Regs.set(X86::RBP); 830 Regs.set(X86::RSI); 831 Regs.set(X86::RDI); 832 Regs.set(X86::RDX); 833 Regs.set(X86::RCX); 834 Regs.set(X86::R8); 835 Regs.set(X86::R9); 836 Regs.set(X86::R10); 837 Regs.set(X86::R11); 838 Regs.set(X86::R12); 839 Regs.set(X86::R13); 840 Regs.set(X86::R14); 841 Regs.set(X86::R15); 842 } 843 844 void getClassicGPRegs(BitVector &Regs) const override { 845 Regs |= getAliases(X86::RAX); 846 Regs |= getAliases(X86::RBX); 847 Regs |= getAliases(X86::RBP); 848 Regs |= getAliases(X86::RSI); 849 Regs |= getAliases(X86::RDI); 850 Regs |= getAliases(X86::RDX); 851 Regs |= getAliases(X86::RCX); 852 } 853 854 void getRepRegs(BitVector &Regs) const override { 855 Regs |= getAliases(X86::RCX); 856 } 857 858 MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override { 859 Reg = getX86SubSuperRegister(Reg, Size * 8); 860 assert((Reg != X86::NoRegister) && "Invalid register"); 861 return Reg; 862 } 863 864 bool isUpper8BitReg(MCPhysReg Reg) const override { 865 switch (Reg) { 866 case X86::AH: 867 case X86::BH: 868 case X86::CH: 869 case X86::DH: 870 return true; 871 default: 872 return false; 873 } 874 } 875 876 bool cannotUseREX(const MCInst &Inst) const override { 877 switch (Inst.getOpcode()) { 878 case X86::MOV8mr_NOREX: 879 case X86::MOV8rm_NOREX: 880 case X86::MOV8rr_NOREX: 881 case X86::MOVSX32rm8_NOREX: 882 case X86::MOVSX32rr8_NOREX: 883 case X86::MOVZX32rm8_NOREX: 884 case X86::MOVZX32rr8_NOREX: 885 case X86::MOV8mr: 886 case X86::MOV8rm: 887 case X86::MOV8rr: 888 case X86::MOVSX32rm8: 889 case X86::MOVSX32rr8: 890 case X86::MOVZX32rm8: 891 case X86::MOVZX32rr8: 892 case X86::TEST8ri: 893 for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) { 894 if (!Operand.isReg()) 895 continue; 896 if (isUpper8BitReg(Operand.getReg())) 897 return true; 898 } 899 [[fallthrough]]; 900 default: 901 return false; 902 } 903 } 904 905 static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) { 906 using namespace llvm::X86; 907 int OpType = getOperandType(Inst.getOpcode(), MemOpNo); 908 return getMemOperandSize(OpType) / 8; 909 } 910 911 /// Classifying a stack access as *not* "SIMPLE" here means we don't know how 912 /// to change this instruction memory access. It will disable any changes to 913 /// the stack layout, so we can't do the most aggressive form of shrink 914 /// wrapping. We must do so in a way that keeps the original stack layout. 915 /// Otherwise you need to adjust the offset of all instructions accessing the 916 /// stack: we can't do that anymore because there is one instruction that is 917 /// not simple. There are other implications as well. We have heuristics to 918 /// detect when a register is callee-saved and thus eligible for shrink 919 /// wrapping. If you are restoring a register using a non-simple stack access, 920 /// then it is classified as NOT callee-saved, and it disables shrink wrapping 921 /// for *that* register (but not for others). 922 /// 923 /// Classifying a stack access as "size 0" or detecting an indexed memory 924 /// access (to address a vector, for example) here means we know there is a 925 /// stack access, but we can't quite understand how wide is the access in 926 /// bytes. This is very serious because we can't understand how memory 927 /// accesses alias with each other for this function. This will essentially 928 /// disable not only shrink wrapping but all frame analysis, it will fail it 929 /// as "we don't understand this function and we give up on it". 930 bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, 931 bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm, 932 uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size, 933 bool &IsSimple, bool &IsIndexed) const override { 934 // Detect simple push/pop cases first 935 if (int Sz = getPushSize(Inst)) { 936 IsLoad = false; 937 IsStore = true; 938 IsStoreFromReg = true; 939 StackPtrReg = X86::RSP; 940 StackOffset = -Sz; 941 Size = Sz; 942 IsSimple = true; 943 if (Inst.getOperand(0).isImm()) 944 SrcImm = Inst.getOperand(0).getImm(); 945 else if (Inst.getOperand(0).isReg()) 946 Reg = Inst.getOperand(0).getReg(); 947 else 948 IsSimple = false; 949 950 return true; 951 } 952 if (int Sz = getPopSize(Inst)) { 953 IsLoad = true; 954 IsStore = false; 955 if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) { 956 IsSimple = false; 957 } else { 958 Reg = Inst.getOperand(0).getReg(); 959 IsSimple = true; 960 } 961 StackPtrReg = X86::RSP; 962 StackOffset = 0; 963 Size = Sz; 964 return true; 965 } 966 967 struct InstInfo { 968 // Size in bytes that Inst loads from memory. 969 uint8_t DataSize; 970 bool IsLoad; 971 bool IsStore; 972 bool StoreFromReg; 973 bool Simple; 974 }; 975 976 InstInfo I; 977 int MemOpNo = getMemoryOperandNo(Inst); 978 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 979 // If it is not dealing with a memory operand, we discard it 980 if (MemOpNo == -1 || MCII.isCall()) 981 return false; 982 983 switch (Inst.getOpcode()) { 984 default: { 985 bool IsLoad = MCII.mayLoad(); 986 bool IsStore = MCII.mayStore(); 987 // Is it LEA? (deals with memory but is not loading nor storing) 988 if (!IsLoad && !IsStore) { 989 I = {0, IsLoad, IsStore, false, false}; 990 break; 991 } 992 uint8_t Sz = getMemDataSize(Inst, MemOpNo); 993 I = {Sz, IsLoad, IsStore, false, false}; 994 break; 995 } 996 // Report simple stack accesses 997 case X86::MOV8rm: I = {1, true, false, false, true}; break; 998 case X86::MOV16rm: I = {2, true, false, false, true}; break; 999 case X86::MOV32rm: I = {4, true, false, false, true}; break; 1000 case X86::MOV64rm: I = {8, true, false, false, true}; break; 1001 case X86::MOV8mr: I = {1, false, true, true, true}; break; 1002 case X86::MOV16mr: I = {2, false, true, true, true}; break; 1003 case X86::MOV32mr: I = {4, false, true, true, true}; break; 1004 case X86::MOV64mr: I = {8, false, true, true, true}; break; 1005 case X86::MOV8mi: I = {1, false, true, false, true}; break; 1006 case X86::MOV16mi: I = {2, false, true, false, true}; break; 1007 case X86::MOV32mi: I = {4, false, true, false, true}; break; 1008 } // end switch (Inst.getOpcode()) 1009 1010 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1011 if (!MO) { 1012 LLVM_DEBUG(dbgs() << "Evaluate failed on "); 1013 LLVM_DEBUG(Inst.dump()); 1014 return false; 1015 } 1016 1017 // Make sure it's a stack access 1018 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) 1019 return false; 1020 1021 IsLoad = I.IsLoad; 1022 IsStore = I.IsStore; 1023 IsStoreFromReg = I.StoreFromReg; 1024 Size = I.DataSize; 1025 IsSimple = I.Simple; 1026 StackPtrReg = MO->BaseRegNum; 1027 StackOffset = MO->DispImm; 1028 IsIndexed = 1029 MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister; 1030 1031 if (!I.Simple) 1032 return true; 1033 1034 // Retrieve related register in simple MOV from/to stack operations. 1035 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 1036 if (I.IsLoad) { 1037 MCOperand RegOpnd = Inst.getOperand(0); 1038 assert(RegOpnd.isReg() && "unexpected destination operand"); 1039 Reg = RegOpnd.getReg(); 1040 } else if (I.IsStore) { 1041 MCOperand SrcOpnd = 1042 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1043 if (I.StoreFromReg) { 1044 assert(SrcOpnd.isReg() && "unexpected source operand"); 1045 Reg = SrcOpnd.getReg(); 1046 } else { 1047 assert(SrcOpnd.isImm() && "unexpected source operand"); 1048 SrcImm = SrcOpnd.getImm(); 1049 } 1050 } 1051 1052 return true; 1053 } 1054 1055 void changeToPushOrPop(MCInst &Inst) const override { 1056 assert(!isPush(Inst) && !isPop(Inst)); 1057 1058 struct InstInfo { 1059 // Size in bytes that Inst loads from memory. 1060 uint8_t DataSize; 1061 bool IsLoad; 1062 bool StoreFromReg; 1063 }; 1064 1065 InstInfo I; 1066 switch (Inst.getOpcode()) { 1067 default: { 1068 llvm_unreachable("Unhandled opcode"); 1069 return; 1070 } 1071 case X86::MOV16rm: I = {2, true, false}; break; 1072 case X86::MOV32rm: I = {4, true, false}; break; 1073 case X86::MOV64rm: I = {8, true, false}; break; 1074 case X86::MOV16mr: I = {2, false, true}; break; 1075 case X86::MOV32mr: I = {4, false, true}; break; 1076 case X86::MOV64mr: I = {8, false, true}; break; 1077 case X86::MOV16mi: I = {2, false, false}; break; 1078 case X86::MOV32mi: I = {4, false, false}; break; 1079 } // end switch (Inst.getOpcode()) 1080 1081 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1082 if (!MO) { 1083 llvm_unreachable("Evaluate failed"); 1084 return; 1085 } 1086 // Make sure it's a stack access 1087 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) { 1088 llvm_unreachable("Not a stack access"); 1089 return; 1090 } 1091 1092 unsigned MemOpOffset = getMemoryOperandNo(Inst); 1093 unsigned NewOpcode = 0; 1094 if (I.IsLoad) { 1095 switch (I.DataSize) { 1096 case 2: NewOpcode = X86::POP16r; break; 1097 case 4: NewOpcode = X86::POP32r; break; 1098 case 8: NewOpcode = X86::POP64r; break; 1099 default: 1100 llvm_unreachable("Unexpected size"); 1101 } 1102 unsigned RegOpndNum = Inst.getOperand(0).getReg(); 1103 Inst.clear(); 1104 Inst.setOpcode(NewOpcode); 1105 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1106 } else { 1107 MCOperand SrcOpnd = 1108 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1109 if (I.StoreFromReg) { 1110 switch (I.DataSize) { 1111 case 2: NewOpcode = X86::PUSH16r; break; 1112 case 4: NewOpcode = X86::PUSH32r; break; 1113 case 8: NewOpcode = X86::PUSH64r; break; 1114 default: 1115 llvm_unreachable("Unexpected size"); 1116 } 1117 assert(SrcOpnd.isReg() && "Unexpected source operand"); 1118 unsigned RegOpndNum = SrcOpnd.getReg(); 1119 Inst.clear(); 1120 Inst.setOpcode(NewOpcode); 1121 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1122 } else { 1123 switch (I.DataSize) { 1124 case 2: NewOpcode = X86::PUSH16i8; break; 1125 case 4: NewOpcode = X86::PUSH32i8; break; 1126 case 8: NewOpcode = X86::PUSH64i32; break; 1127 default: 1128 llvm_unreachable("Unexpected size"); 1129 } 1130 assert(SrcOpnd.isImm() && "Unexpected source operand"); 1131 int64_t SrcImm = SrcOpnd.getImm(); 1132 Inst.clear(); 1133 Inst.setOpcode(NewOpcode); 1134 Inst.addOperand(MCOperand::createImm(SrcImm)); 1135 } 1136 } 1137 } 1138 1139 bool isStackAdjustment(const MCInst &Inst) const override { 1140 switch (Inst.getOpcode()) { 1141 default: 1142 return false; 1143 case X86::SUB64ri32: 1144 case X86::SUB64ri8: 1145 case X86::ADD64ri32: 1146 case X86::ADD64ri8: 1147 case X86::LEA64r: 1148 break; 1149 } 1150 1151 return any_of(defOperands(Inst), [](const MCOperand &Op) { 1152 return Op.isReg() && Op.getReg() == X86::RSP; 1153 }); 1154 } 1155 1156 bool 1157 evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output, 1158 std::pair<MCPhysReg, int64_t> Input1, 1159 std::pair<MCPhysReg, int64_t> Input2) const override { 1160 1161 auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> { 1162 if (Reg == Input1.first) 1163 return Input1.second; 1164 if (Reg == Input2.first) 1165 return Input2.second; 1166 return make_error_code(errc::result_out_of_range); 1167 }; 1168 1169 switch (Inst.getOpcode()) { 1170 default: 1171 return false; 1172 1173 case X86::SUB64ri32: 1174 case X86::SUB64ri8: 1175 if (!Inst.getOperand(2).isImm()) 1176 return false; 1177 if (ErrorOr<int64_t> InputVal = 1178 getOperandVal(Inst.getOperand(1).getReg())) 1179 Output = *InputVal - Inst.getOperand(2).getImm(); 1180 else 1181 return false; 1182 break; 1183 case X86::ADD64ri32: 1184 case X86::ADD64ri8: 1185 if (!Inst.getOperand(2).isImm()) 1186 return false; 1187 if (ErrorOr<int64_t> InputVal = 1188 getOperandVal(Inst.getOperand(1).getReg())) 1189 Output = *InputVal + Inst.getOperand(2).getImm(); 1190 else 1191 return false; 1192 break; 1193 case X86::ADD64i32: 1194 if (!Inst.getOperand(0).isImm()) 1195 return false; 1196 if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX)) 1197 Output = *InputVal + Inst.getOperand(0).getImm(); 1198 else 1199 return false; 1200 break; 1201 1202 case X86::LEA64r: { 1203 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1204 if (!MO) 1205 return false; 1206 1207 if (MO->BaseRegNum == X86::NoRegister || 1208 MO->IndexRegNum != X86::NoRegister || 1209 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 1210 return false; 1211 1212 if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum)) 1213 Output = *InputVal + MO->DispImm; 1214 else 1215 return false; 1216 1217 break; 1218 } 1219 } 1220 return true; 1221 } 1222 1223 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, 1224 MCPhysReg &To) const override { 1225 switch (Inst.getOpcode()) { 1226 default: 1227 return false; 1228 case X86::LEAVE: 1229 case X86::LEAVE64: 1230 To = getStackPointer(); 1231 From = getFramePointer(); 1232 return true; 1233 case X86::MOV64rr: 1234 To = Inst.getOperand(0).getReg(); 1235 From = Inst.getOperand(1).getReg(); 1236 return true; 1237 } 1238 } 1239 1240 MCPhysReg getStackPointer() const override { return X86::RSP; } 1241 MCPhysReg getFramePointer() const override { return X86::RBP; } 1242 MCPhysReg getFlagsReg() const override { return X86::EFLAGS; } 1243 1244 bool escapesVariable(const MCInst &Inst, 1245 bool HasFramePointer) const override { 1246 int MemOpNo = getMemoryOperandNo(Inst); 1247 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 1248 const unsigned NumDefs = MCII.getNumDefs(); 1249 static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |= 1250 getAliases(X86::RBP)); 1251 static BitVector SPAliases(getAliases(X86::RSP)); 1252 1253 // FIXME: PUSH can be technically a leak, but let's ignore this for now 1254 // because a lot of harmless prologue code will spill SP to the stack. 1255 // Unless push is clearly pushing an object address to the stack as 1256 // demonstrated by having a MemOp. 1257 bool IsPush = isPush(Inst); 1258 if (IsPush && MemOpNo == -1) 1259 return false; 1260 1261 // We use this to detect LEA (has memop but does not access mem) 1262 bool AccessMem = MCII.mayLoad() || MCII.mayStore(); 1263 bool DoesLeak = false; 1264 for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) { 1265 // Ignore if SP/BP is used to dereference memory -- that's fine 1266 if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo && 1267 I <= MemOpNo + 5) 1268 continue; 1269 // Ignore if someone is writing to SP/BP 1270 if (I < static_cast<int>(NumDefs)) 1271 continue; 1272 1273 const MCOperand &Operand = Inst.getOperand(I); 1274 if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) { 1275 DoesLeak = true; 1276 break; 1277 } 1278 if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) { 1279 DoesLeak = true; 1280 break; 1281 } 1282 } 1283 1284 // If potential leak, check if it is not just writing to itself/sp/bp 1285 if (DoesLeak) { 1286 DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) { 1287 assert(Operand.isReg()); 1288 MCPhysReg Reg = Operand.getReg(); 1289 return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg]; 1290 }); 1291 } 1292 return DoesLeak; 1293 } 1294 1295 bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override { 1296 unsigned ImmOpNo = -1U; 1297 int MemOpNo = getMemoryOperandNo(Inst); 1298 if (MemOpNo != -1) 1299 ImmOpNo = MemOpNo + X86::AddrDisp; 1300 else 1301 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 1302 ++Index) 1303 if (Inst.getOperand(Index).isImm()) 1304 ImmOpNo = Index; 1305 if (ImmOpNo == -1U) 1306 return false; 1307 1308 MCOperand &Operand = Inst.getOperand(ImmOpNo); 1309 Amt += Operand.getImm(); 1310 Operand.setImm(Amt); 1311 // Check for the need for relaxation 1312 if (int64_t(Amt) == int64_t(int8_t(Amt))) 1313 return true; 1314 1315 // Relax instruction 1316 switch (Inst.getOpcode()) { 1317 case X86::SUB64ri8: 1318 Inst.setOpcode(X86::SUB64ri32); 1319 break; 1320 case X86::ADD64ri8: 1321 Inst.setOpcode(X86::ADD64ri32); 1322 break; 1323 default: 1324 // No need for relaxation 1325 break; 1326 } 1327 return true; 1328 } 1329 1330 /// TODO: this implementation currently works for the most common opcodes that 1331 /// load from memory. It can be extended to work with memory store opcodes as 1332 /// well as more memory load opcodes. 1333 bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, 1334 uint64_t Offset) const override { 1335 enum CheckSignExt : uint8_t { 1336 NOCHECK = 0, 1337 CHECK8, 1338 CHECK32, 1339 }; 1340 1341 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 1342 struct InstInfo { 1343 // Size in bytes that Inst loads from memory. 1344 uint8_t DataSize; 1345 1346 // True when the target operand has to be duplicated because the opcode 1347 // expects a LHS operand. 1348 bool HasLHS; 1349 1350 // List of checks and corresponding opcodes to be used. We try to use the 1351 // smallest possible immediate value when various sizes are available, 1352 // hence we may need to check whether a larger constant fits in a smaller 1353 // immediate. 1354 CheckList Checks; 1355 }; 1356 1357 InstInfo I; 1358 1359 switch (Inst.getOpcode()) { 1360 default: { 1361 switch (getPopSize(Inst)) { 1362 case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1363 case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1364 case 8: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1365 {NOCHECK, X86::MOV64rm}}}; break; 1366 default: return false; 1367 } 1368 break; 1369 } 1370 1371 // MOV 1372 case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 1373 case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1374 case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1375 case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1376 {NOCHECK, X86::MOV64rm}}}; break; 1377 1378 // MOVZX 1379 case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 1380 case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 1381 case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 1382 1383 // CMP 1384 case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 1385 case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8}, 1386 {NOCHECK, X86::CMP16ri}}}; break; 1387 case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8}, 1388 {NOCHECK, X86::CMP32ri}}}; break; 1389 case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8}, 1390 {CHECK32, X86::CMP64ri32}, 1391 {NOCHECK, X86::CMP64rm}}}; break; 1392 1393 // TEST 1394 case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 1395 case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 1396 case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 1397 case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 1398 {NOCHECK, X86::TEST64mr}}}; break; 1399 1400 // ADD 1401 case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 1402 case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8}, 1403 {NOCHECK, X86::ADD16ri}}}; break; 1404 case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8}, 1405 {NOCHECK, X86::ADD32ri}}}; break; 1406 case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8}, 1407 {CHECK32, X86::ADD64ri32}, 1408 {NOCHECK, X86::ADD64rm}}}; break; 1409 1410 // SUB 1411 case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 1412 case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8}, 1413 {NOCHECK, X86::SUB16ri}}}; break; 1414 case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8}, 1415 {NOCHECK, X86::SUB32ri}}}; break; 1416 case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8}, 1417 {CHECK32, X86::SUB64ri32}, 1418 {NOCHECK, X86::SUB64rm}}}; break; 1419 1420 // AND 1421 case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 1422 case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8}, 1423 {NOCHECK, X86::AND16ri}}}; break; 1424 case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8}, 1425 {NOCHECK, X86::AND32ri}}}; break; 1426 case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8}, 1427 {CHECK32, X86::AND64ri32}, 1428 {NOCHECK, X86::AND64rm}}}; break; 1429 1430 // OR 1431 case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 1432 case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8}, 1433 {NOCHECK, X86::OR16ri}}}; break; 1434 case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8}, 1435 {NOCHECK, X86::OR32ri}}}; break; 1436 case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8}, 1437 {CHECK32, X86::OR64ri32}, 1438 {NOCHECK, X86::OR64rm}}}; break; 1439 1440 // XOR 1441 case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 1442 case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8}, 1443 {NOCHECK, X86::XOR16ri}}}; break; 1444 case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8}, 1445 {NOCHECK, X86::XOR32ri}}}; break; 1446 case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8}, 1447 {CHECK32, X86::XOR64ri32}, 1448 {NOCHECK, X86::XOR64rm}}}; break; 1449 } 1450 1451 // Compute the immediate value. 1452 assert(Offset + I.DataSize <= ConstantData.size() && 1453 "invalid offset for given constant data"); 1454 int64_t ImmVal = 1455 DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize); 1456 1457 // Compute the new opcode. 1458 unsigned NewOpcode = 0; 1459 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 1460 NewOpcode = Check.second; 1461 if (Check.first == NOCHECK) 1462 break; 1463 if (Check.first == CHECK8 && isInt<8>(ImmVal)) 1464 break; 1465 if (Check.first == CHECK32 && isInt<32>(ImmVal)) 1466 break; 1467 } 1468 if (NewOpcode == Inst.getOpcode()) 1469 return false; 1470 1471 // Modify the instruction. 1472 MCOperand ImmOp = MCOperand::createImm(ImmVal); 1473 uint32_t TargetOpNum = 0; 1474 // Test instruction does not follow the regular pattern of putting the 1475 // memory reference of a load (5 MCOperands) last in the list of operands. 1476 // Since it is not modifying the register operand, it is not treated as 1477 // a destination operand and it is not the first operand as it is in the 1478 // other instructions we treat here. 1479 if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri || 1480 NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32) 1481 TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands; 1482 1483 MCOperand TargetOp = Inst.getOperand(TargetOpNum); 1484 Inst.clear(); 1485 Inst.setOpcode(NewOpcode); 1486 Inst.addOperand(TargetOp); 1487 if (I.HasLHS) 1488 Inst.addOperand(TargetOp); 1489 Inst.addOperand(ImmOp); 1490 1491 return true; 1492 } 1493 1494 /// TODO: this implementation currently works for the most common opcodes that 1495 /// load from memory. It can be extended to work with memory store opcodes as 1496 /// well as more memory load opcodes. 1497 bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override { 1498 unsigned NewOpcode; 1499 1500 switch (Inst.getOpcode()) { 1501 default: { 1502 switch (getPopSize(Inst)) { 1503 case 2: NewOpcode = X86::MOV16rr; break; 1504 case 4: NewOpcode = X86::MOV32rr; break; 1505 case 8: NewOpcode = X86::MOV64rr; break; 1506 default: return false; 1507 } 1508 break; 1509 } 1510 1511 // MOV 1512 case X86::MOV8rm: NewOpcode = X86::MOV8rr; break; 1513 case X86::MOV16rm: NewOpcode = X86::MOV16rr; break; 1514 case X86::MOV32rm: NewOpcode = X86::MOV32rr; break; 1515 case X86::MOV64rm: NewOpcode = X86::MOV64rr; break; 1516 } 1517 1518 // Modify the instruction. 1519 MCOperand RegOp = MCOperand::createReg(RegNum); 1520 MCOperand TargetOp = Inst.getOperand(0); 1521 Inst.clear(); 1522 Inst.setOpcode(NewOpcode); 1523 Inst.addOperand(TargetOp); 1524 Inst.addOperand(RegOp); 1525 1526 return true; 1527 } 1528 1529 bool isRedundantMove(const MCInst &Inst) const override { 1530 switch (Inst.getOpcode()) { 1531 default: 1532 return false; 1533 1534 // MOV 1535 case X86::MOV8rr: 1536 case X86::MOV16rr: 1537 case X86::MOV32rr: 1538 case X86::MOV64rr: 1539 break; 1540 } 1541 1542 assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg()); 1543 return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg(); 1544 } 1545 1546 bool requiresAlignedAddress(const MCInst &Inst) const override { 1547 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 1548 for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) { 1549 const MCOperandInfo &Op = Desc.operands()[I]; 1550 if (Op.OperandType != MCOI::OPERAND_REGISTER) 1551 continue; 1552 if (Op.RegClass == X86::VR128RegClassID) 1553 return true; 1554 } 1555 return false; 1556 } 1557 1558 bool convertJmpToTailCall(MCInst &Inst) override { 1559 if (isTailCall(Inst)) 1560 return false; 1561 1562 int NewOpcode; 1563 switch (Inst.getOpcode()) { 1564 default: 1565 return false; 1566 case X86::JMP_1: 1567 case X86::JMP_2: 1568 case X86::JMP_4: 1569 NewOpcode = X86::JMP_4; 1570 break; 1571 case X86::JMP16m: 1572 case X86::JMP32m: 1573 case X86::JMP64m: 1574 NewOpcode = X86::JMP32m; 1575 break; 1576 case X86::JMP16r: 1577 case X86::JMP32r: 1578 case X86::JMP64r: 1579 NewOpcode = X86::JMP32r; 1580 break; 1581 } 1582 1583 Inst.setOpcode(NewOpcode); 1584 setTailCall(Inst); 1585 return true; 1586 } 1587 1588 bool convertTailCallToJmp(MCInst &Inst) override { 1589 int NewOpcode; 1590 switch (Inst.getOpcode()) { 1591 default: 1592 return false; 1593 case X86::JMP_4: 1594 NewOpcode = X86::JMP_1; 1595 break; 1596 case X86::JMP32m: 1597 NewOpcode = X86::JMP64m; 1598 break; 1599 case X86::JMP32r: 1600 NewOpcode = X86::JMP64r; 1601 break; 1602 } 1603 1604 Inst.setOpcode(NewOpcode); 1605 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1606 clearOffset(Inst); 1607 return true; 1608 } 1609 1610 bool convertTailCallToCall(MCInst &Inst) override { 1611 int NewOpcode; 1612 switch (Inst.getOpcode()) { 1613 default: 1614 return false; 1615 case X86::JMP_4: 1616 NewOpcode = X86::CALL64pcrel32; 1617 break; 1618 case X86::JMP32m: 1619 NewOpcode = X86::CALL64m; 1620 break; 1621 case X86::JMP32r: 1622 NewOpcode = X86::CALL64r; 1623 break; 1624 } 1625 1626 Inst.setOpcode(NewOpcode); 1627 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1628 return true; 1629 } 1630 1631 bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation, 1632 MCContext *Ctx) override { 1633 assert((Inst.getOpcode() == X86::CALL64pcrel32 || 1634 (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) && 1635 "64-bit direct (tail) call instruction expected"); 1636 const auto NewOpcode = 1637 (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m; 1638 Inst.setOpcode(NewOpcode); 1639 1640 // Replace the first operand and preserve auxiliary operands of 1641 // the instruction. 1642 Inst.erase(Inst.begin()); 1643 Inst.insert(Inst.begin(), 1644 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 1645 Inst.insert(Inst.begin(), 1646 MCOperand::createExpr( // Displacement 1647 MCSymbolRefExpr::create(TargetLocation, 1648 MCSymbolRefExpr::VK_None, *Ctx))); 1649 Inst.insert(Inst.begin(), 1650 MCOperand::createReg(X86::NoRegister)); // IndexReg 1651 Inst.insert(Inst.begin(), 1652 MCOperand::createImm(1)); // ScaleAmt 1653 Inst.insert(Inst.begin(), 1654 MCOperand::createReg(X86::RIP)); // BaseReg 1655 1656 return true; 1657 } 1658 1659 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { 1660 bool IsTailCall = isTailCall(Inst); 1661 if (IsTailCall) 1662 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1663 if (Inst.getOpcode() == X86::CALL64m || 1664 (Inst.getOpcode() == X86::JMP32m && IsTailCall)) { 1665 Inst.setOpcode(X86::MOV64rm); 1666 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1667 return; 1668 } 1669 if (Inst.getOpcode() == X86::CALL64r || 1670 (Inst.getOpcode() == X86::JMP32r && IsTailCall)) { 1671 Inst.setOpcode(X86::MOV64rr); 1672 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1673 return; 1674 } 1675 LLVM_DEBUG(Inst.dump()); 1676 llvm_unreachable("not implemented"); 1677 } 1678 1679 bool shortenInstruction(MCInst &Inst, 1680 const MCSubtargetInfo &STI) const override { 1681 unsigned OldOpcode = Inst.getOpcode(); 1682 unsigned NewOpcode = OldOpcode; 1683 1684 int MemOpNo = getMemoryOperandNo(Inst); 1685 1686 // Check and remove redundant Address-Size override prefix. 1687 if (opts::X86StripRedundantAddressSize) { 1688 uint64_t TSFlags = Info->get(OldOpcode).TSFlags; 1689 unsigned Flags = Inst.getFlags(); 1690 1691 if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) && 1692 Flags & X86::IP_HAS_AD_SIZE) 1693 Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE); 1694 } 1695 1696 // Check and remove EIZ/RIZ. These cases represent ambiguous cases where 1697 // SIB byte is present, but no index is used and modrm alone should have 1698 // been enough. Converting to NoRegister effectively removes the SIB byte. 1699 if (MemOpNo >= 0) { 1700 MCOperand &IndexOp = 1701 Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg); 1702 if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ) 1703 IndexOp = MCOperand::createReg(X86::NoRegister); 1704 } 1705 1706 if (isBranch(Inst)) { 1707 NewOpcode = getShortBranchOpcode(OldOpcode); 1708 } else if (OldOpcode == X86::MOV64ri) { 1709 if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1710 const int64_t Imm = 1711 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1712 if (int64_t(Imm) == int64_t(int32_t(Imm))) 1713 NewOpcode = X86::MOV64ri32; 1714 } 1715 } else { 1716 // If it's arithmetic instruction check if signed operand fits in 1 byte. 1717 const unsigned ShortOpcode = getShortArithOpcode(OldOpcode); 1718 if (ShortOpcode != OldOpcode && 1719 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1720 int64_t Imm = 1721 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1722 if (int64_t(Imm) == int64_t(int8_t(Imm))) 1723 NewOpcode = ShortOpcode; 1724 } 1725 } 1726 1727 if (NewOpcode == OldOpcode) 1728 return false; 1729 1730 Inst.setOpcode(NewOpcode); 1731 return true; 1732 } 1733 1734 bool 1735 convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp, 1736 bool AllowBasePtrStackMemOp) const override { 1737 // - Register-register moves are OK 1738 // - Stores are filtered out by opcode (no store CMOV) 1739 // - Non-stack loads are prohibited (generally unsafe) 1740 // - Stack loads are OK if AllowStackMemOp is true 1741 // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true 1742 if (isLoad(Inst)) { 1743 // If stack memory operands are not allowed, no loads are allowed 1744 if (!AllowStackMemOp) 1745 return false; 1746 1747 // If stack memory operands are allowed, check if it's a load from stack 1748 bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed; 1749 MCPhysReg Reg; 1750 int32_t SrcImm; 1751 uint16_t StackPtrReg; 1752 int64_t StackOffset; 1753 uint8_t Size; 1754 bool IsStackAccess = 1755 isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, 1756 StackPtrReg, StackOffset, Size, IsSimple, IsIndexed); 1757 // Prohibit non-stack-based loads 1758 if (!IsStackAccess) 1759 return false; 1760 // If stack memory operands are allowed, check if it's RBP-based 1761 if (!AllowBasePtrStackMemOp && 1762 RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg)) 1763 return false; 1764 } 1765 1766 unsigned NewOpcode = 0; 1767 switch (Inst.getOpcode()) { 1768 case X86::MOV16rr: 1769 NewOpcode = X86::CMOV16rr; 1770 break; 1771 case X86::MOV16rm: 1772 NewOpcode = X86::CMOV16rm; 1773 break; 1774 case X86::MOV32rr: 1775 NewOpcode = X86::CMOV32rr; 1776 break; 1777 case X86::MOV32rm: 1778 NewOpcode = X86::CMOV32rm; 1779 break; 1780 case X86::MOV64rr: 1781 NewOpcode = X86::CMOV64rr; 1782 break; 1783 case X86::MOV64rm: 1784 NewOpcode = X86::CMOV64rm; 1785 break; 1786 default: 1787 return false; 1788 } 1789 Inst.setOpcode(NewOpcode); 1790 // Insert CC at the end of prime operands, before annotations 1791 Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst), 1792 MCOperand::createImm(CC)); 1793 // CMOV is a 3-operand MCInst, so duplicate the destination as src1 1794 Inst.insert(Inst.begin(), Inst.getOperand(0)); 1795 return true; 1796 } 1797 1798 bool lowerTailCall(MCInst &Inst) override { 1799 if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) { 1800 Inst.setOpcode(X86::JMP_1); 1801 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1802 return true; 1803 } 1804 return false; 1805 } 1806 1807 const MCSymbol *getTargetSymbol(const MCInst &Inst, 1808 unsigned OpNum = 0) const override { 1809 if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) 1810 return nullptr; 1811 1812 const MCOperand &Op = Inst.getOperand(OpNum); 1813 if (!Op.isExpr()) 1814 return nullptr; 1815 1816 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 1817 if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) 1818 return nullptr; 1819 1820 return &SymExpr->getSymbol(); 1821 } 1822 1823 // This is the same as the base class, but since we are overriding one of 1824 // getTargetSymbol's signatures above, we need to override all of them. 1825 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { 1826 return &cast<const MCSymbolRefExpr>(Expr)->getSymbol(); 1827 } 1828 1829 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, 1830 const MCSymbol *&TBB, const MCSymbol *&FBB, 1831 MCInst *&CondBranch, 1832 MCInst *&UncondBranch) const override { 1833 auto I = End; 1834 1835 // Bottom-up analysis 1836 while (I != Begin) { 1837 --I; 1838 1839 // Ignore nops and CFIs 1840 if (isPseudo(*I)) 1841 continue; 1842 1843 // Stop when we find the first non-terminator 1844 if (!isTerminator(*I)) 1845 break; 1846 1847 if (!isBranch(*I)) 1848 break; 1849 1850 // Handle unconditional branches. 1851 if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 || 1852 I->getOpcode() == X86::JMP_4) && 1853 !isTailCall(*I)) { 1854 // If any code was seen after this unconditional branch, we've seen 1855 // unreachable code. Ignore them. 1856 CondBranch = nullptr; 1857 UncondBranch = &*I; 1858 const MCSymbol *Sym = getTargetSymbol(*I); 1859 assert(Sym != nullptr && 1860 "Couldn't extract BB symbol from jump operand"); 1861 TBB = Sym; 1862 continue; 1863 } 1864 1865 // Handle conditional branches and ignore indirect branches 1866 if (!isUnsupportedBranch(I->getOpcode()) && 1867 getCondCode(*I) == X86::COND_INVALID) { 1868 // Indirect branch 1869 return false; 1870 } 1871 1872 if (CondBranch == nullptr) { 1873 const MCSymbol *TargetBB = getTargetSymbol(*I); 1874 if (TargetBB == nullptr) { 1875 // Unrecognized branch target 1876 return false; 1877 } 1878 FBB = TBB; 1879 TBB = TargetBB; 1880 CondBranch = &*I; 1881 continue; 1882 } 1883 1884 llvm_unreachable("multiple conditional branches in one BB"); 1885 } 1886 return true; 1887 } 1888 1889 template <typename Itr> 1890 std::pair<IndirectBranchType, MCInst *> 1891 analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const { 1892 // Analyze PIC-style jump table code template: 1893 // 1894 // lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr 1895 // mov ({%r1|%r2}, %index, 4), {%r2|%r1} 1896 // add %r2, %r1 1897 // jmp *%r1 1898 // 1899 // (with any irrelevant instructions in-between) 1900 // 1901 // When we call this helper we've already determined %r1 and %r2, and 1902 // reverse instruction iterator \p II is pointing to the ADD instruction. 1903 // 1904 // PIC jump table looks like following: 1905 // 1906 // JT: ---------- 1907 // E1:| L1 - JT | 1908 // |----------| 1909 // E2:| L2 - JT | 1910 // |----------| 1911 // | | 1912 // ...... 1913 // En:| Ln - JT | 1914 // ---------- 1915 // 1916 // Where L1, L2, ..., Ln represent labels in the function. 1917 // 1918 // The actual relocations in the table will be of the form: 1919 // 1920 // Ln - JT 1921 // = (Ln - En) + (En - JT) 1922 // = R_X86_64_PC32(Ln) + En - JT 1923 // = R_X86_64_PC32(Ln + offsetof(En)) 1924 // 1925 LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n"); 1926 MCInst *MemLocInstr = nullptr; 1927 const MCInst *MovInstr = nullptr; 1928 while (++II != IE) { 1929 MCInst &Instr = *II; 1930 const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode()); 1931 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) && 1932 !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) { 1933 // Ignore instructions that don't affect R1, R2 registers. 1934 continue; 1935 } 1936 if (!MovInstr) { 1937 // Expect to see MOV instruction. 1938 if (!isMOVSX64rm32(Instr)) { 1939 LLVM_DEBUG(dbgs() << "MOV instruction expected.\n"); 1940 break; 1941 } 1942 1943 // Check if it's setting %r1 or %r2. In canonical form it sets %r2. 1944 // If it sets %r1 - rename the registers so we have to only check 1945 // a single form. 1946 unsigned MovDestReg = Instr.getOperand(0).getReg(); 1947 if (MovDestReg != R2) 1948 std::swap(R1, R2); 1949 if (MovDestReg != R2) { 1950 LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n"); 1951 break; 1952 } 1953 1954 // Verify operands for MOV. 1955 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1956 if (!MO) 1957 break; 1958 if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 || 1959 MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 || 1960 MO->SegRegNum != X86::NoRegister) 1961 break; 1962 MovInstr = &Instr; 1963 } else { 1964 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo)) 1965 continue; 1966 if (!isLEA64r(Instr)) { 1967 LLVM_DEBUG(dbgs() << "LEA instruction expected\n"); 1968 break; 1969 } 1970 if (Instr.getOperand(0).getReg() != R1) { 1971 LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n"); 1972 break; 1973 } 1974 1975 // Verify operands for LEA. 1976 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1977 if (!MO) 1978 break; 1979 if (MO->BaseRegNum != RegInfo->getProgramCounter() || 1980 MO->IndexRegNum != X86::NoRegister || 1981 MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr) 1982 break; 1983 MemLocInstr = &Instr; 1984 break; 1985 } 1986 } 1987 1988 if (!MemLocInstr) 1989 return std::make_pair(IndirectBranchType::UNKNOWN, nullptr); 1990 1991 LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n"); 1992 return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE, 1993 MemLocInstr); 1994 } 1995 1996 IndirectBranchType analyzeIndirectBranch( 1997 MCInst &Instruction, InstructionIterator Begin, InstructionIterator End, 1998 const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut, 1999 unsigned &IndexRegNumOut, int64_t &DispValueOut, 2000 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override { 2001 // Try to find a (base) memory location from where the address for 2002 // the indirect branch is loaded. For X86-64 the memory will be specified 2003 // in the following format: 2004 // 2005 // {%rip}/{%basereg} + Imm + IndexReg * Scale 2006 // 2007 // We are interested in the cases where Scale == sizeof(uintptr_t) and 2008 // the contents of the memory are presumably an array of pointers to code. 2009 // 2010 // Normal jump table: 2011 // 2012 // jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr 2013 // 2014 // or 2015 // 2016 // mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr 2017 // ... 2018 // jmp %r1 2019 // 2020 // We handle PIC-style jump tables separately. 2021 // 2022 MemLocInstrOut = nullptr; 2023 BaseRegNumOut = X86::NoRegister; 2024 IndexRegNumOut = X86::NoRegister; 2025 DispValueOut = 0; 2026 DispExprOut = nullptr; 2027 2028 std::reverse_iterator<InstructionIterator> II(End); 2029 std::reverse_iterator<InstructionIterator> IE(Begin); 2030 2031 IndirectBranchType Type = IndirectBranchType::UNKNOWN; 2032 2033 // An instruction referencing memory used by jump instruction (directly or 2034 // via register). This location could be an array of function pointers 2035 // in case of indirect tail call, or a jump table. 2036 MCInst *MemLocInstr = nullptr; 2037 2038 if (MCPlus::getNumPrimeOperands(Instruction) == 1) { 2039 // If the indirect jump is on register - try to detect if the 2040 // register value is loaded from a memory location. 2041 assert(Instruction.getOperand(0).isReg() && "register operand expected"); 2042 const unsigned R1 = Instruction.getOperand(0).getReg(); 2043 // Check if one of the previous instructions defines the jump-on register. 2044 for (auto PrevII = II; PrevII != IE; ++PrevII) { 2045 MCInst &PrevInstr = *PrevII; 2046 const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode()); 2047 2048 if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo)) 2049 continue; 2050 2051 if (isMoveMem2Reg(PrevInstr)) { 2052 MemLocInstr = &PrevInstr; 2053 break; 2054 } 2055 if (isADD64rr(PrevInstr)) { 2056 unsigned R2 = PrevInstr.getOperand(2).getReg(); 2057 if (R1 == R2) 2058 return IndirectBranchType::UNKNOWN; 2059 std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2); 2060 break; 2061 } 2062 return IndirectBranchType::UNKNOWN; 2063 } 2064 if (!MemLocInstr) { 2065 // No definition seen for the register in this function so far. Could be 2066 // an input parameter - which means it is an external code reference. 2067 // It also could be that the definition happens to be in the code that 2068 // we haven't processed yet. Since we have to be conservative, return 2069 // as UNKNOWN case. 2070 return IndirectBranchType::UNKNOWN; 2071 } 2072 } else { 2073 MemLocInstr = &Instruction; 2074 } 2075 2076 const MCRegister RIPRegister = RegInfo->getProgramCounter(); 2077 2078 // Analyze the memory location. 2079 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr); 2080 if (!MO) 2081 return IndirectBranchType::UNKNOWN; 2082 2083 BaseRegNumOut = MO->BaseRegNum; 2084 IndexRegNumOut = MO->IndexRegNum; 2085 DispValueOut = MO->DispImm; 2086 DispExprOut = MO->DispExpr; 2087 2088 if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) || 2089 MO->SegRegNum != X86::NoRegister) 2090 return IndirectBranchType::UNKNOWN; 2091 2092 if (MemLocInstr == &Instruction && 2093 (!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) { 2094 MemLocInstrOut = MemLocInstr; 2095 return IndirectBranchType::POSSIBLE_FIXED_BRANCH; 2096 } 2097 2098 if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2099 (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)) 2100 return IndirectBranchType::UNKNOWN; 2101 2102 if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2103 MO->ScaleImm != PtrSize) 2104 return IndirectBranchType::UNKNOWN; 2105 2106 MemLocInstrOut = MemLocInstr; 2107 2108 return Type; 2109 } 2110 2111 /// Analyze a callsite to see if it could be a virtual method call. This only 2112 /// checks to see if the overall pattern is satisfied, it does not guarantee 2113 /// that the callsite is a true virtual method call. 2114 /// The format of virtual method calls that are recognized is one of the 2115 /// following: 2116 /// 2117 /// Form 1: (found in debug code) 2118 /// add METHOD_OFFSET, %VtableReg 2119 /// mov (%VtableReg), %MethodReg 2120 /// ... 2121 /// call or jmp *%MethodReg 2122 /// 2123 /// Form 2: 2124 /// mov METHOD_OFFSET(%VtableReg), %MethodReg 2125 /// ... 2126 /// call or jmp *%MethodReg 2127 /// 2128 /// Form 3: 2129 /// ... 2130 /// call or jmp *METHOD_OFFSET(%VtableReg) 2131 /// 2132 bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin, 2133 InstructionIterator ForwardEnd, 2134 std::vector<MCInst *> &MethodFetchInsns, 2135 unsigned &VtableRegNum, unsigned &MethodRegNum, 2136 uint64_t &MethodOffset) const override { 2137 VtableRegNum = X86::NoRegister; 2138 MethodRegNum = X86::NoRegister; 2139 MethodOffset = 0; 2140 2141 std::reverse_iterator<InstructionIterator> Itr(ForwardEnd); 2142 std::reverse_iterator<InstructionIterator> End(ForwardBegin); 2143 2144 MCInst &CallInst = *Itr++; 2145 assert(isIndirectBranch(CallInst) || isCall(CallInst)); 2146 2147 // The call can just be jmp offset(reg) 2148 if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) { 2149 if (!MO->DispExpr && MO->BaseRegNum != X86::RIP && 2150 MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) { 2151 MethodRegNum = MO->BaseRegNum; 2152 if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister && 2153 MO->SegRegNum == X86::NoRegister) { 2154 VtableRegNum = MethodRegNum; 2155 MethodOffset = MO->DispImm; 2156 MethodFetchInsns.push_back(&CallInst); 2157 return true; 2158 } 2159 } 2160 return false; 2161 } 2162 if (CallInst.getOperand(0).isReg()) 2163 MethodRegNum = CallInst.getOperand(0).getReg(); 2164 else 2165 return false; 2166 2167 if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) { 2168 VtableRegNum = X86::NoRegister; 2169 MethodRegNum = X86::NoRegister; 2170 return false; 2171 } 2172 2173 // find load from vtable, this may or may not include the method offset 2174 while (Itr != End) { 2175 MCInst &CurInst = *Itr++; 2176 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2177 if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) { 2178 if (!isLoad(CurInst)) 2179 return false; 2180 if (std::optional<X86MemOperand> MO = 2181 evaluateX86MemoryOperand(CurInst)) { 2182 if (!MO->DispExpr && MO->ScaleImm == 1 && 2183 MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP && 2184 MO->BaseRegNum != X86::NoRegister && 2185 MO->IndexRegNum == X86::NoRegister && 2186 MO->SegRegNum == X86::NoRegister && MO->BaseRegNum != X86::RIP) { 2187 VtableRegNum = MO->BaseRegNum; 2188 MethodOffset = MO->DispImm; 2189 MethodFetchInsns.push_back(&CurInst); 2190 if (MethodOffset != 0) 2191 return true; 2192 break; 2193 } 2194 } 2195 return false; 2196 } 2197 } 2198 2199 if (!VtableRegNum) 2200 return false; 2201 2202 // look for any adds affecting the method register. 2203 while (Itr != End) { 2204 MCInst &CurInst = *Itr++; 2205 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2206 if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) { 2207 if (isADDri(CurInst)) { 2208 assert(!MethodOffset); 2209 MethodOffset = CurInst.getOperand(2).getImm(); 2210 MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst); 2211 break; 2212 } 2213 } 2214 } 2215 2216 return true; 2217 } 2218 2219 bool createStackPointerIncrement(MCInst &Inst, int Size, 2220 bool NoFlagsClobber) const override { 2221 if (NoFlagsClobber) { 2222 Inst.setOpcode(X86::LEA64r); 2223 Inst.clear(); 2224 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2225 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2226 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2227 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2228 Inst.addOperand(MCOperand::createImm(-Size)); // Displacement 2229 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2230 return true; 2231 } 2232 Inst.setOpcode(X86::SUB64ri8); 2233 Inst.clear(); 2234 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2235 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2236 Inst.addOperand(MCOperand::createImm(Size)); 2237 return true; 2238 } 2239 2240 bool createStackPointerDecrement(MCInst &Inst, int Size, 2241 bool NoFlagsClobber) const override { 2242 if (NoFlagsClobber) { 2243 Inst.setOpcode(X86::LEA64r); 2244 Inst.clear(); 2245 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2246 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2247 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2248 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2249 Inst.addOperand(MCOperand::createImm(Size)); // Displacement 2250 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2251 return true; 2252 } 2253 Inst.setOpcode(X86::ADD64ri8); 2254 Inst.clear(); 2255 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2256 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2257 Inst.addOperand(MCOperand::createImm(Size)); 2258 return true; 2259 } 2260 2261 bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset, 2262 const MCPhysReg &SrcReg, int Size) const override { 2263 unsigned NewOpcode; 2264 switch (Size) { 2265 default: 2266 return false; 2267 case 2: NewOpcode = X86::MOV16mr; break; 2268 case 4: NewOpcode = X86::MOV32mr; break; 2269 case 8: NewOpcode = X86::MOV64mr; break; 2270 } 2271 Inst.setOpcode(NewOpcode); 2272 Inst.clear(); 2273 Inst.addOperand(MCOperand::createReg(StackReg)); // BaseReg 2274 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2275 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2276 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2277 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2278 Inst.addOperand(MCOperand::createReg(SrcReg)); 2279 return true; 2280 } 2281 2282 bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, 2283 int Offset, const MCPhysReg &DstReg, 2284 int Size) const override { 2285 return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister, 2286 Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister, 2287 DstReg, Size); 2288 } 2289 2290 bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale, 2291 const MCPhysReg &IndexReg, int64_t Offset, 2292 const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg, 2293 const MCPhysReg &DstReg, int Size) const override { 2294 unsigned NewOpcode; 2295 switch (Size) { 2296 default: 2297 return false; 2298 case 2: NewOpcode = X86::MOV16rm; break; 2299 case 4: NewOpcode = X86::MOV32rm; break; 2300 case 8: NewOpcode = X86::MOV64rm; break; 2301 } 2302 Inst.setOpcode(NewOpcode); 2303 Inst.clear(); 2304 Inst.addOperand(MCOperand::createReg(DstReg)); 2305 Inst.addOperand(MCOperand::createReg(BaseReg)); 2306 Inst.addOperand(MCOperand::createImm(Scale)); 2307 Inst.addOperand(MCOperand::createReg(IndexReg)); 2308 if (OffsetExpr) 2309 Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement 2310 else 2311 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2312 Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg 2313 return true; 2314 } 2315 2316 void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest, 2317 uint32_t Imm) const override { 2318 Inst.setOpcode(X86::MOV64ri32); 2319 Inst.clear(); 2320 Inst.addOperand(MCOperand::createReg(Dest)); 2321 Inst.addOperand(MCOperand::createImm(Imm)); 2322 } 2323 2324 bool createIncMemory(MCInst &Inst, const MCSymbol *Target, 2325 MCContext *Ctx) const override { 2326 2327 Inst.setOpcode(X86::LOCK_INC64m); 2328 Inst.clear(); 2329 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 2330 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2331 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2332 2333 Inst.addOperand(MCOperand::createExpr( 2334 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, 2335 *Ctx))); // Displacement 2336 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2337 return true; 2338 } 2339 2340 bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts, 2341 const MCOperand &BaseReg, const MCOperand &Scale, 2342 const MCOperand &IndexReg, const MCOperand &Offset, 2343 const MCOperand &TmpReg) const override { 2344 // The code fragment we emit here is: 2345 // 2346 // mov32 (%base, %index, scale), %tmpreg 2347 // ijmp *(%tmpreg) 2348 // 2349 MCInst IJmp; 2350 IJmp.setOpcode(X86::JMP64r); 2351 IJmp.addOperand(TmpReg); 2352 2353 MCInst Load; 2354 Load.setOpcode(X86::MOV32rm); 2355 Load.addOperand(TmpReg); 2356 Load.addOperand(BaseReg); 2357 Load.addOperand(Scale); 2358 Load.addOperand(IndexReg); 2359 Load.addOperand(Offset); 2360 Load.addOperand(MCOperand::createReg(X86::NoRegister)); 2361 2362 Insts.push_back(Load); 2363 Insts.push_back(IJmp); 2364 return true; 2365 } 2366 2367 bool createNoop(MCInst &Inst) const override { 2368 Inst.setOpcode(X86::NOOP); 2369 return true; 2370 } 2371 2372 bool createReturn(MCInst &Inst) const override { 2373 Inst.setOpcode(X86::RET64); 2374 return true; 2375 } 2376 2377 InstructionListType createInlineMemcpy(bool ReturnEnd) const override { 2378 InstructionListType Code; 2379 if (ReturnEnd) 2380 Code.emplace_back(MCInstBuilder(X86::LEA64r) 2381 .addReg(X86::RAX) 2382 .addReg(X86::RDI) 2383 .addImm(1) 2384 .addReg(X86::RDX) 2385 .addImm(0) 2386 .addReg(X86::NoRegister)); 2387 else 2388 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2389 .addReg(X86::RAX) 2390 .addReg(X86::RDI)); 2391 2392 Code.emplace_back(MCInstBuilder(X86::MOV32rr) 2393 .addReg(X86::ECX) 2394 .addReg(X86::EDX)); 2395 Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64)); 2396 2397 return Code; 2398 } 2399 2400 InstructionListType createOneByteMemcpy() const override { 2401 InstructionListType Code; 2402 Code.emplace_back(MCInstBuilder(X86::MOV8rm) 2403 .addReg(X86::CL) 2404 .addReg(X86::RSI) 2405 .addImm(0) 2406 .addReg(X86::NoRegister) 2407 .addImm(0) 2408 .addReg(X86::NoRegister)); 2409 Code.emplace_back(MCInstBuilder(X86::MOV8mr) 2410 .addReg(X86::RDI) 2411 .addImm(0) 2412 .addReg(X86::NoRegister) 2413 .addImm(0) 2414 .addReg(X86::NoRegister) 2415 .addReg(X86::CL)); 2416 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2417 .addReg(X86::RAX) 2418 .addReg(X86::RDI)); 2419 return Code; 2420 } 2421 2422 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, 2423 const MCSymbol *Target, 2424 MCContext *Ctx) const override { 2425 InstructionListType Code; 2426 Code.emplace_back(MCInstBuilder(X86::CMP64ri8) 2427 .addReg(RegNo) 2428 .addImm(Imm)); 2429 Code.emplace_back(MCInstBuilder(X86::JCC_1) 2430 .addExpr(MCSymbolRefExpr::create( 2431 Target, MCSymbolRefExpr::VK_None, *Ctx)) 2432 .addImm(X86::COND_E)); 2433 return Code; 2434 } 2435 2436 std::optional<Relocation> 2437 createRelocation(const MCFixup &Fixup, 2438 const MCAsmBackend &MAB) const override { 2439 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); 2440 2441 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); 2442 const uint64_t RelOffset = Fixup.getOffset(); 2443 2444 uint64_t RelType; 2445 if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) { 2446 switch (FKI.TargetSize) { 2447 default: 2448 return std::nullopt; 2449 case 8: RelType = ELF::R_X86_64_PC8; break; 2450 case 16: RelType = ELF::R_X86_64_PC16; break; 2451 case 32: RelType = ELF::R_X86_64_PC32; break; 2452 case 64: RelType = ELF::R_X86_64_PC64; break; 2453 } 2454 } else { 2455 switch (FKI.TargetSize) { 2456 default: 2457 return std::nullopt; 2458 case 8: RelType = ELF::R_X86_64_8; break; 2459 case 16: RelType = ELF::R_X86_64_16; break; 2460 case 32: RelType = ELF::R_X86_64_32; break; 2461 case 64: RelType = ELF::R_X86_64_64; break; 2462 } 2463 } 2464 2465 // Extract a symbol and an addend out of the fixup value expression. 2466 // 2467 // Only the following limited expression types are supported: 2468 // Symbol + Addend 2469 // Symbol 2470 uint64_t Addend = 0; 2471 MCSymbol *Symbol = nullptr; 2472 const MCExpr *ValueExpr = Fixup.getValue(); 2473 if (ValueExpr->getKind() == MCExpr::Binary) { 2474 const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr); 2475 assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add && 2476 "unexpected binary expression"); 2477 const MCExpr *LHS = BinaryExpr->getLHS(); 2478 assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS"); 2479 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS)); 2480 const MCExpr *RHS = BinaryExpr->getRHS(); 2481 assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS"); 2482 Addend = cast<MCConstantExpr>(RHS)->getValue(); 2483 } else { 2484 assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value"); 2485 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr)); 2486 } 2487 2488 return Relocation({RelOffset, Symbol, RelType, Addend, 0}); 2489 } 2490 2491 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, 2492 int64_t Addend, MCContext *Ctx, int64_t &Value, 2493 uint64_t RelType) const override { 2494 unsigned ImmOpNo = -1U; 2495 2496 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 2497 ++Index) { 2498 if (Inst.getOperand(Index).isImm()) { 2499 ImmOpNo = Index; 2500 // TODO: this is a bit hacky. It finds the correct operand by 2501 // searching for a specific immediate value. If no value is 2502 // provided it defaults to the last immediate operand found. 2503 // This could lead to unexpected results if the instruction 2504 // has more than one immediate with the same value. 2505 if (Inst.getOperand(ImmOpNo).getImm() == Value) 2506 break; 2507 } 2508 } 2509 2510 if (ImmOpNo == -1U) 2511 return false; 2512 2513 Value = Inst.getOperand(ImmOpNo).getImm(); 2514 2515 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType); 2516 2517 return true; 2518 } 2519 2520 bool replaceRegWithImm(MCInst &Inst, unsigned Register, 2521 int64_t Imm) const override { 2522 2523 enum CheckSignExt : uint8_t { 2524 NOCHECK = 0, 2525 CHECK8, 2526 CHECK32, 2527 }; 2528 2529 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 2530 struct InstInfo { 2531 // Size in bytes that Inst loads from memory. 2532 uint8_t DataSize; 2533 2534 // True when the target operand has to be duplicated because the opcode 2535 // expects a LHS operand. 2536 bool HasLHS; 2537 2538 // List of checks and corresponding opcodes to be used. We try to use the 2539 // smallest possible immediate value when various sizes are available, 2540 // hence we may need to check whether a larger constant fits in a smaller 2541 // immediate. 2542 CheckList Checks; 2543 }; 2544 2545 InstInfo I; 2546 2547 switch (Inst.getOpcode()) { 2548 default: { 2549 switch (getPushSize(Inst)) { 2550 2551 case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break; 2552 case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break; 2553 case 8: I = {8, false, {{CHECK8, X86::PUSH64i8}, 2554 {CHECK32, X86::PUSH64i32}, 2555 {NOCHECK, Inst.getOpcode()}}}; break; 2556 default: return false; 2557 } 2558 break; 2559 } 2560 2561 // MOV 2562 case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 2563 case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 2564 case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 2565 case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32}, 2566 {NOCHECK, X86::MOV64ri}}}; break; 2567 2568 case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break; 2569 case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break; 2570 case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break; 2571 case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32}, 2572 {NOCHECK, X86::MOV64mr}}}; break; 2573 2574 // MOVZX 2575 case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 2576 case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 2577 case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 2578 2579 // CMP 2580 case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 2581 case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8}, 2582 {NOCHECK, X86::CMP16ri}}}; break; 2583 case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8}, 2584 {NOCHECK, X86::CMP32ri}}}; break; 2585 case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8}, 2586 {CHECK32, X86::CMP64ri32}, 2587 {NOCHECK, X86::CMP64rr}}}; break; 2588 2589 // TEST 2590 case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 2591 case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 2592 case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 2593 case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 2594 {NOCHECK, X86::TEST64rr}}}; break; 2595 2596 // ADD 2597 case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 2598 case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8}, 2599 {NOCHECK, X86::ADD16ri}}}; break; 2600 case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8}, 2601 {NOCHECK, X86::ADD32ri}}}; break; 2602 case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8}, 2603 {CHECK32, X86::ADD64ri32}, 2604 {NOCHECK, X86::ADD64rr}}}; break; 2605 2606 // SUB 2607 case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 2608 case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8}, 2609 {NOCHECK, X86::SUB16ri}}}; break; 2610 case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8}, 2611 {NOCHECK, X86::SUB32ri}}}; break; 2612 case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8}, 2613 {CHECK32, X86::SUB64ri32}, 2614 {NOCHECK, X86::SUB64rr}}}; break; 2615 2616 // AND 2617 case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 2618 case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8}, 2619 {NOCHECK, X86::AND16ri}}}; break; 2620 case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8}, 2621 {NOCHECK, X86::AND32ri}}}; break; 2622 case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8}, 2623 {CHECK32, X86::AND64ri32}, 2624 {NOCHECK, X86::AND64rr}}}; break; 2625 2626 // OR 2627 case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 2628 case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8}, 2629 {NOCHECK, X86::OR16ri}}}; break; 2630 case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8}, 2631 {NOCHECK, X86::OR32ri}}}; break; 2632 case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8}, 2633 {CHECK32, X86::OR64ri32}, 2634 {NOCHECK, X86::OR64rr}}}; break; 2635 2636 // XOR 2637 case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 2638 case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8}, 2639 {NOCHECK, X86::XOR16ri}}}; break; 2640 case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8}, 2641 {NOCHECK, X86::XOR32ri}}}; break; 2642 case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8}, 2643 {CHECK32, X86::XOR64ri32}, 2644 {NOCHECK, X86::XOR64rr}}}; break; 2645 } 2646 2647 // Compute the new opcode. 2648 unsigned NewOpcode = 0; 2649 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 2650 NewOpcode = Check.second; 2651 if (Check.first == NOCHECK) 2652 break; 2653 if (Check.first == CHECK8 && isInt<8>(Imm)) 2654 break; 2655 if (Check.first == CHECK32 && isInt<32>(Imm)) 2656 break; 2657 } 2658 if (NewOpcode == Inst.getOpcode()) 2659 return false; 2660 2661 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2662 2663 unsigned NumFound = 0; 2664 for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0), 2665 E = InstDesc.getNumOperands(); 2666 Index != E; ++Index) 2667 if (Inst.getOperand(Index).isReg() && 2668 Inst.getOperand(Index).getReg() == Register) 2669 NumFound++; 2670 2671 if (NumFound != 1) 2672 return false; 2673 2674 MCOperand TargetOp = Inst.getOperand(0); 2675 Inst.clear(); 2676 Inst.setOpcode(NewOpcode); 2677 Inst.addOperand(TargetOp); 2678 if (I.HasLHS) 2679 Inst.addOperand(TargetOp); 2680 Inst.addOperand(MCOperand::createImm(Imm)); 2681 2682 return true; 2683 } 2684 2685 bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace, 2686 unsigned ReplaceWith) const override { 2687 2688 // Get the HasLHS value so that iteration can be done 2689 bool HasLHS; 2690 if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) || 2691 X86::isSUB(Inst.getOpcode())) { 2692 HasLHS = true; 2693 } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) || 2694 X86::isTEST(Inst.getOpcode())) { 2695 HasLHS = false; 2696 } else { 2697 switch (Inst.getOpcode()) { 2698 case X86::MOV8rr: 2699 case X86::MOV8rm: 2700 case X86::MOV8mr: 2701 case X86::MOV8ri: 2702 case X86::MOV16rr: 2703 case X86::MOV16rm: 2704 case X86::MOV16mr: 2705 case X86::MOV16ri: 2706 case X86::MOV32rr: 2707 case X86::MOV32rm: 2708 case X86::MOV32mr: 2709 case X86::MOV32ri: 2710 case X86::MOV64rr: 2711 case X86::MOV64rm: 2712 case X86::MOV64mr: 2713 case X86::MOV64ri: 2714 case X86::MOVZX16rr8: 2715 case X86::MOVZX32rr8: 2716 case X86::MOVZX32rr16: 2717 case X86::MOVSX32rm8: 2718 case X86::MOVSX32rr8: 2719 case X86::MOVSX64rm32: 2720 case X86::LEA64r: 2721 HasLHS = false; 2722 break; 2723 default: 2724 return false; 2725 } 2726 } 2727 2728 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2729 2730 bool FoundOne = false; 2731 2732 // Iterate only through src operands that arent also dest operands 2733 for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0), 2734 E = InstDesc.getNumOperands(); 2735 Index != E; ++Index) { 2736 BitVector RegAliases = getAliases(ToReplace, true); 2737 if (!Inst.getOperand(Index).isReg() || 2738 !RegAliases.test(Inst.getOperand(Index).getReg())) 2739 continue; 2740 // Resize register if needed 2741 unsigned SizedReplaceWith = getAliasSized( 2742 ReplaceWith, getRegSize(Inst.getOperand(Index).getReg())); 2743 MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith); 2744 Inst.getOperand(Index) = NewOperand; 2745 FoundOne = true; 2746 } 2747 2748 // Return true if at least one operand was replaced 2749 return FoundOne; 2750 } 2751 2752 bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, 2753 MCContext *Ctx) const override { 2754 Inst.setOpcode(X86::JMP_1); 2755 Inst.addOperand(MCOperand::createExpr( 2756 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx))); 2757 return true; 2758 } 2759 2760 bool createCall(MCInst &Inst, const MCSymbol *Target, 2761 MCContext *Ctx) override { 2762 Inst.setOpcode(X86::CALL64pcrel32); 2763 Inst.addOperand(MCOperand::createExpr( 2764 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2765 return true; 2766 } 2767 2768 bool createTailCall(MCInst &Inst, const MCSymbol *Target, 2769 MCContext *Ctx) override { 2770 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); 2771 } 2772 2773 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, 2774 MCContext *Ctx) override { 2775 Seq.clear(); 2776 Seq.emplace_back(); 2777 createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true); 2778 } 2779 2780 bool createTrap(MCInst &Inst) const override { 2781 Inst.clear(); 2782 Inst.setOpcode(X86::TRAP); 2783 return true; 2784 } 2785 2786 bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, 2787 MCContext *Ctx) const override { 2788 unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); 2789 assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); 2790 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); 2791 Inst.getOperand(0) = MCOperand::createExpr( 2792 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2793 return true; 2794 } 2795 2796 bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx, 2797 unsigned CC) const override { 2798 if (CC == X86::COND_INVALID) 2799 return false; 2800 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC); 2801 Inst.getOperand(0) = MCOperand::createExpr( 2802 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2803 return true; 2804 } 2805 2806 unsigned getCanonicalBranchCondCode(unsigned CC) const override { 2807 switch (CC) { 2808 default: return X86::COND_INVALID; 2809 2810 case X86::COND_E: return X86::COND_E; 2811 case X86::COND_NE: return X86::COND_E; 2812 2813 case X86::COND_L: return X86::COND_L; 2814 case X86::COND_GE: return X86::COND_L; 2815 2816 case X86::COND_LE: return X86::COND_G; 2817 case X86::COND_G: return X86::COND_G; 2818 2819 case X86::COND_B: return X86::COND_B; 2820 case X86::COND_AE: return X86::COND_B; 2821 2822 case X86::COND_BE: return X86::COND_A; 2823 case X86::COND_A: return X86::COND_A; 2824 2825 case X86::COND_S: return X86::COND_S; 2826 case X86::COND_NS: return X86::COND_S; 2827 2828 case X86::COND_P: return X86::COND_P; 2829 case X86::COND_NP: return X86::COND_P; 2830 2831 case X86::COND_O: return X86::COND_O; 2832 case X86::COND_NO: return X86::COND_O; 2833 } 2834 } 2835 2836 bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, 2837 MCContext *Ctx) const override { 2838 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && 2839 "Invalid instruction"); 2840 Inst.getOperand(0) = MCOperand::createExpr( 2841 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2842 return true; 2843 } 2844 2845 MCPhysReg getX86R11() const override { return X86::R11; } 2846 2847 unsigned getShortBranchOpcode(unsigned Opcode) const override { 2848 switch (Opcode) { 2849 default: 2850 return Opcode; 2851 case X86::JMP_2: 2852 return X86::JMP_1; 2853 case X86::JMP_4: 2854 return X86::JMP_1; 2855 case X86::JCC_2: 2856 return X86::JCC_1; 2857 case X86::JCC_4: 2858 return X86::JCC_1; 2859 } 2860 } 2861 2862 MCPhysReg getIntArgRegister(unsigned ArgNo) const override { 2863 // FIXME: this should depend on the calling convention. 2864 switch (ArgNo) { 2865 case 0: return X86::RDI; 2866 case 1: return X86::RSI; 2867 case 2: return X86::RDX; 2868 case 3: return X86::RCX; 2869 case 4: return X86::R8; 2870 case 5: return X86::R9; 2871 default: return getNoRegister(); 2872 } 2873 } 2874 2875 void createPause(MCInst &Inst) const override { 2876 Inst.clear(); 2877 Inst.setOpcode(X86::PAUSE); 2878 } 2879 2880 void createLfence(MCInst &Inst) const override { 2881 Inst.clear(); 2882 Inst.setOpcode(X86::LFENCE); 2883 } 2884 2885 bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, 2886 bool IsTailCall) override { 2887 Inst.clear(); 2888 Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 2889 Inst.addOperand(MCOperand::createExpr( 2890 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2891 if (IsTailCall) 2892 setTailCall(Inst); 2893 return true; 2894 } 2895 2896 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, 2897 MCContext *Ctx, bool IsTailCall) override { 2898 Seq.clear(); 2899 MCInst Inst; 2900 Inst.setOpcode(X86::JMP_1); 2901 Inst.addOperand(MCOperand::createExpr( 2902 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2903 if (IsTailCall) 2904 setTailCall(Inst); 2905 Seq.emplace_back(Inst); 2906 } 2907 2908 bool isConditionalMove(const MCInst &Inst) const override { 2909 unsigned OpCode = Inst.getOpcode(); 2910 return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr || 2911 OpCode == X86::CMOV64rr); 2912 } 2913 2914 bool isBranchOnMem(const MCInst &Inst) const override { 2915 unsigned OpCode = Inst.getOpcode(); 2916 if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) || 2917 OpCode == X86::JMP64m) 2918 return true; 2919 2920 return false; 2921 } 2922 2923 bool isBranchOnReg(const MCInst &Inst) const override { 2924 unsigned OpCode = Inst.getOpcode(); 2925 if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) || 2926 OpCode == X86::JMP64r) 2927 return true; 2928 2929 return false; 2930 } 2931 2932 void createPushRegister(MCInst &Inst, MCPhysReg Reg, 2933 unsigned Size) const override { 2934 Inst.clear(); 2935 unsigned NewOpcode = 0; 2936 if (Reg == X86::EFLAGS) { 2937 switch (Size) { 2938 case 2: NewOpcode = X86::PUSHF16; break; 2939 case 4: NewOpcode = X86::PUSHF32; break; 2940 case 8: NewOpcode = X86::PUSHF64; break; 2941 default: 2942 llvm_unreachable("Unexpected size"); 2943 } 2944 Inst.setOpcode(NewOpcode); 2945 return; 2946 } 2947 switch (Size) { 2948 case 2: NewOpcode = X86::PUSH16r; break; 2949 case 4: NewOpcode = X86::PUSH32r; break; 2950 case 8: NewOpcode = X86::PUSH64r; break; 2951 default: 2952 llvm_unreachable("Unexpected size"); 2953 } 2954 Inst.setOpcode(NewOpcode); 2955 Inst.addOperand(MCOperand::createReg(Reg)); 2956 } 2957 2958 void createPopRegister(MCInst &Inst, MCPhysReg Reg, 2959 unsigned Size) const override { 2960 Inst.clear(); 2961 unsigned NewOpcode = 0; 2962 if (Reg == X86::EFLAGS) { 2963 switch (Size) { 2964 case 2: NewOpcode = X86::POPF16; break; 2965 case 4: NewOpcode = X86::POPF32; break; 2966 case 8: NewOpcode = X86::POPF64; break; 2967 default: 2968 llvm_unreachable("Unexpected size"); 2969 } 2970 Inst.setOpcode(NewOpcode); 2971 return; 2972 } 2973 switch (Size) { 2974 case 2: NewOpcode = X86::POP16r; break; 2975 case 4: NewOpcode = X86::POP32r; break; 2976 case 8: NewOpcode = X86::POP64r; break; 2977 default: 2978 llvm_unreachable("Unexpected size"); 2979 } 2980 Inst.setOpcode(NewOpcode); 2981 Inst.addOperand(MCOperand::createReg(Reg)); 2982 } 2983 2984 void createPushFlags(MCInst &Inst, unsigned Size) const override { 2985 return createPushRegister(Inst, X86::EFLAGS, Size); 2986 } 2987 2988 void createPopFlags(MCInst &Inst, unsigned Size) const override { 2989 return createPopRegister(Inst, X86::EFLAGS, Size); 2990 } 2991 2992 void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value, 2993 unsigned Size) const { 2994 unsigned int Opcode; 2995 switch (Size) { 2996 case 1: Opcode = X86::ADD8ri; break; 2997 case 2: Opcode = X86::ADD16ri; break; 2998 case 4: Opcode = X86::ADD32ri; break; 2999 default: 3000 llvm_unreachable("Unexpected size"); 3001 } 3002 Inst.setOpcode(Opcode); 3003 Inst.clear(); 3004 Inst.addOperand(MCOperand::createReg(Reg)); 3005 Inst.addOperand(MCOperand::createReg(Reg)); 3006 Inst.addOperand(MCOperand::createImm(Value)); 3007 } 3008 3009 void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg, 3010 unsigned Size) const { 3011 unsigned int Opcode; 3012 switch (Size) { 3013 case 1: Opcode = X86::MOV8ri; break; 3014 case 2: Opcode = X86::MOV16ri; break; 3015 case 4: Opcode = X86::MOV32ri; break; 3016 // Writing to a 32-bit register always zeros the upper 32 bits of the 3017 // full-width register 3018 case 8: 3019 Opcode = X86::MOV32ri; 3020 Reg = getAliasSized(Reg, 4); 3021 break; 3022 default: 3023 llvm_unreachable("Unexpected size"); 3024 } 3025 Inst.setOpcode(Opcode); 3026 Inst.clear(); 3027 Inst.addOperand(MCOperand::createReg(Reg)); 3028 Inst.addOperand(MCOperand::createImm(0)); 3029 } 3030 3031 void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const { 3032 Inst.setOpcode(X86::SETCCr); 3033 Inst.clear(); 3034 Inst.addOperand(MCOperand::createReg(Reg)); 3035 Inst.addOperand(MCOperand::createImm(X86::COND_O)); 3036 } 3037 3038 void createX86Lahf(MCInst &Inst) const { 3039 Inst.setOpcode(X86::LAHF); 3040 Inst.clear(); 3041 } 3042 3043 void createX86Sahf(MCInst &Inst) const { 3044 Inst.setOpcode(X86::SAHF); 3045 Inst.clear(); 3046 } 3047 3048 InstructionListType createInstrIncMemory(const MCSymbol *Target, 3049 MCContext *Ctx, 3050 bool IsLeaf) const override { 3051 InstructionListType Instrs(IsLeaf ? 13 : 11); 3052 unsigned int I = 0; 3053 3054 // Don't clobber application red zone (ABI dependent) 3055 if (IsLeaf) 3056 createStackPointerIncrement(Instrs[I++], 128, 3057 /*NoFlagsClobber=*/true); 3058 3059 // Performance improvements based on the optimization discussed at 3060 // https://reviews.llvm.org/D6629 3061 // LAHF/SAHF are used instead of PUSHF/POPF 3062 // PUSHF 3063 createPushRegister(Instrs[I++], X86::RAX, 8); 3064 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3065 createX86Lahf(Instrs[I++]); 3066 createPushRegister(Instrs[I++], X86::RAX, 8); 3067 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3068 createX86SaveOVFlagToRegister(Instrs[I++], X86::AL); 3069 // LOCK INC 3070 createIncMemory(Instrs[I++], Target, Ctx); 3071 // POPF 3072 createAddRegImm(Instrs[I++], X86::AL, 127, 1); 3073 createPopRegister(Instrs[I++], X86::RAX, 8); 3074 createX86Sahf(Instrs[I++]); 3075 createPopRegister(Instrs[I++], X86::RAX, 8); 3076 3077 if (IsLeaf) 3078 createStackPointerDecrement(Instrs[I], 128, 3079 /*NoFlagsClobber=*/true); 3080 return Instrs; 3081 } 3082 3083 void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg, 3084 int64_t Disp) const { 3085 Inst.setOpcode(X86::XCHG64rm); 3086 Inst.addOperand(MCOperand::createReg(Source)); 3087 Inst.addOperand(MCOperand::createReg(Source)); 3088 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3089 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3090 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3091 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3092 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3093 } 3094 3095 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg, 3096 int64_t Disp) const { 3097 Inst.setOpcode(X86::JMP64m); 3098 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3099 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3100 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3101 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3102 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3103 } 3104 3105 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, 3106 MCSymbol *HandlerFuncAddr, 3107 int CallSiteID, 3108 MCContext *Ctx) override { 3109 // Check if the target address expression used in the original indirect call 3110 // uses the stack pointer, which we are going to clobber. 3111 static BitVector SPAliases(getAliases(X86::RSP)); 3112 bool UsesSP = any_of(useOperands(CallInst), [&](const MCOperand &Op) { 3113 return Op.isReg() && SPAliases[Op.getReg()]; 3114 }); 3115 3116 InstructionListType Insts; 3117 MCPhysReg TempReg = getIntArgRegister(0); 3118 // Code sequence used to enter indirect call instrumentation helper: 3119 // push %rdi 3120 // add $8, %rsp ;; $rsp may be used in target, so fix it to prev val 3121 // movq target, %rdi ;; via convertIndirectCallTargetToLoad 3122 // sub $8, %rsp ;; restore correct stack value 3123 // push %rdi 3124 // movq $CallSiteID, %rdi 3125 // push %rdi 3126 // callq/jmp HandlerFuncAddr 3127 Insts.emplace_back(); 3128 createPushRegister(Insts.back(), TempReg, 8); 3129 if (UsesSP) { // Only adjust SP if we really need to 3130 Insts.emplace_back(); 3131 createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3132 } 3133 Insts.emplace_back(CallInst); 3134 // Insts.back() and CallInst now share the same annotation instruction. 3135 // Strip it from Insts.back(), only preserving tail call annotation. 3136 stripAnnotations(Insts.back(), /*KeepTC=*/true); 3137 convertIndirectCallToLoad(Insts.back(), TempReg); 3138 if (UsesSP) { 3139 Insts.emplace_back(); 3140 createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3141 } 3142 Insts.emplace_back(); 3143 createPushRegister(Insts.back(), TempReg, 8); 3144 Insts.emplace_back(); 3145 createLoadImmediate(Insts.back(), TempReg, CallSiteID); 3146 Insts.emplace_back(); 3147 createPushRegister(Insts.back(), TempReg, 8); 3148 3149 MCInst &NewCallInst = Insts.emplace_back(); 3150 createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst)); 3151 3152 // Carry over metadata including tail call marker if present. 3153 stripAnnotations(NewCallInst); 3154 moveAnnotations(std::move(CallInst), NewCallInst); 3155 3156 return Insts; 3157 } 3158 3159 InstructionListType createInstrumentedIndCallHandlerExitBB() const override { 3160 const MCPhysReg TempReg = getIntArgRegister(0); 3161 // We just need to undo the sequence created for every ind call in 3162 // instrumentIndirectTarget(), which can be accomplished minimally with: 3163 // popfq 3164 // pop %rdi 3165 // add $16, %rsp 3166 // xchg (%rsp), %rdi 3167 // jmp *-8(%rsp) 3168 InstructionListType Insts(5); 3169 createPopFlags(Insts[0], 8); 3170 createPopRegister(Insts[1], TempReg, 8); 3171 createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false); 3172 createSwap(Insts[3], TempReg, X86::RSP, 0); 3173 createIndirectBranch(Insts[4], X86::RSP, -8); 3174 return Insts; 3175 } 3176 3177 InstructionListType 3178 createInstrumentedIndTailCallHandlerExitBB() const override { 3179 const MCPhysReg TempReg = getIntArgRegister(0); 3180 // Same thing as above, but for tail calls 3181 // popfq 3182 // add $16, %rsp 3183 // pop %rdi 3184 // jmp *-16(%rsp) 3185 InstructionListType Insts(4); 3186 createPopFlags(Insts[0], 8); 3187 createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false); 3188 createPopRegister(Insts[2], TempReg, 8); 3189 createIndirectBranch(Insts[3], X86::RSP, -16); 3190 return Insts; 3191 } 3192 3193 InstructionListType 3194 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, 3195 const MCSymbol *IndCallHandler, 3196 MCContext *Ctx) override { 3197 const MCPhysReg TempReg = getIntArgRegister(0); 3198 // Code sequence used to check whether InstrTampoline was initialized 3199 // and call it if so, returns via IndCallHandler. 3200 // pushfq 3201 // mov InstrTrampoline,%rdi 3202 // cmp $0x0,%rdi 3203 // je IndCallHandler 3204 // callq *%rdi 3205 // jmpq IndCallHandler 3206 InstructionListType Insts; 3207 Insts.emplace_back(); 3208 createPushFlags(Insts.back(), 8); 3209 Insts.emplace_back(); 3210 createMove(Insts.back(), InstrTrampoline, TempReg, Ctx); 3211 InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); 3212 Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); 3213 Insts.emplace_back(); 3214 Insts.back().setOpcode(X86::CALL64r); 3215 Insts.back().addOperand(MCOperand::createReg(TempReg)); 3216 Insts.emplace_back(); 3217 createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); 3218 return Insts; 3219 } 3220 3221 InstructionListType createNumCountersGetter(MCContext *Ctx) const override { 3222 InstructionListType Insts(2); 3223 MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters"); 3224 createMove(Insts[0], NumLocs, X86::EAX, Ctx); 3225 createReturn(Insts[1]); 3226 return Insts; 3227 } 3228 3229 InstructionListType 3230 createInstrLocationsGetter(MCContext *Ctx) const override { 3231 InstructionListType Insts(2); 3232 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations"); 3233 createLea(Insts[0], Locs, X86::EAX, Ctx); 3234 createReturn(Insts[1]); 3235 return Insts; 3236 } 3237 3238 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { 3239 InstructionListType Insts(2); 3240 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables"); 3241 createLea(Insts[0], Locs, X86::EAX, Ctx); 3242 createReturn(Insts[1]); 3243 return Insts; 3244 } 3245 3246 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { 3247 InstructionListType Insts(2); 3248 MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs"); 3249 createMove(Insts[0], NumFuncs, X86::EAX, Ctx); 3250 createReturn(Insts[1]); 3251 return Insts; 3252 } 3253 3254 InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, 3255 MCContext *Ctx) const override { 3256 InstructionListType Insts(1); 3257 createUncondBranch(Insts[0], TgtSym, Ctx); 3258 return Insts; 3259 } 3260 3261 InstructionListType createDummyReturnFunction(MCContext *Ctx) const override { 3262 InstructionListType Insts(1); 3263 createReturn(Insts[0]); 3264 return Insts; 3265 } 3266 3267 BlocksVectorTy indirectCallPromotion( 3268 const MCInst &CallInst, 3269 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3270 const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms, 3271 const std::vector<MCInst *> &MethodFetchInsns, 3272 const bool MinimizeCodeSize, MCContext *Ctx) override { 3273 const bool IsTailCall = isTailCall(CallInst); 3274 const bool IsJumpTable = getJumpTable(CallInst) != 0; 3275 BlocksVectorTy Results; 3276 3277 // Label for the current code block. 3278 MCSymbol *NextTarget = nullptr; 3279 3280 // The join block which contains all the instructions following CallInst. 3281 // MergeBlock remains null if CallInst is a tail call. 3282 MCSymbol *MergeBlock = nullptr; 3283 3284 unsigned FuncAddrReg = X86::R10; 3285 3286 const bool LoadElim = !VtableSyms.empty(); 3287 assert((!LoadElim || VtableSyms.size() == Targets.size()) && 3288 "There must be a vtable entry for every method " 3289 "in the targets vector."); 3290 3291 if (MinimizeCodeSize && !LoadElim) { 3292 std::set<unsigned> UsedRegs; 3293 3294 for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) { 3295 const MCOperand &Op = CallInst.getOperand(I); 3296 if (Op.isReg()) 3297 UsedRegs.insert(Op.getReg()); 3298 } 3299 3300 if (UsedRegs.count(X86::R10) == 0) 3301 FuncAddrReg = X86::R10; 3302 else if (UsedRegs.count(X86::R11) == 0) 3303 FuncAddrReg = X86::R11; 3304 else 3305 return Results; 3306 } 3307 3308 const auto jumpToMergeBlock = [&](InstructionListType &NewCall) { 3309 assert(MergeBlock); 3310 NewCall.push_back(CallInst); 3311 MCInst &Merge = NewCall.back(); 3312 Merge.clear(); 3313 createUncondBranch(Merge, MergeBlock, Ctx); 3314 }; 3315 3316 for (unsigned int i = 0; i < Targets.size(); ++i) { 3317 Results.emplace_back(NextTarget, InstructionListType()); 3318 InstructionListType *NewCall = &Results.back().second; 3319 3320 if (MinimizeCodeSize && !LoadElim) { 3321 // Load the call target into FuncAddrReg. 3322 NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc 3323 MCInst &Target = NewCall->back(); 3324 Target.clear(); 3325 Target.setOpcode(X86::MOV64ri32); 3326 Target.addOperand(MCOperand::createReg(FuncAddrReg)); 3327 if (Targets[i].first) { 3328 // Is this OK? 3329 Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3330 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3331 } else { 3332 const uint64_t Addr = Targets[i].second; 3333 // Immediate address is out of sign extended 32 bit range. 3334 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3335 return BlocksVectorTy(); 3336 3337 Target.addOperand(MCOperand::createImm(Addr)); 3338 } 3339 3340 // Compare current call target to a specific address. 3341 NewCall->push_back(CallInst); 3342 MCInst &Compare = NewCall->back(); 3343 Compare.clear(); 3344 if (isBranchOnReg(CallInst)) 3345 Compare.setOpcode(X86::CMP64rr); 3346 else if (CallInst.getOpcode() == X86::CALL64pcrel32) 3347 Compare.setOpcode(X86::CMP64ri32); 3348 else 3349 Compare.setOpcode(X86::CMP64rm); 3350 3351 Compare.addOperand(MCOperand::createReg(FuncAddrReg)); 3352 3353 // TODO: Would be preferable to only load this value once. 3354 for (unsigned i = 0; 3355 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3356 if (!CallInst.getOperand(i).isInst()) 3357 Compare.addOperand(CallInst.getOperand(i)); 3358 } else { 3359 // Compare current call target to a specific address. 3360 NewCall->push_back(CallInst); 3361 MCInst &Compare = NewCall->back(); 3362 Compare.clear(); 3363 if (isBranchOnReg(CallInst)) 3364 Compare.setOpcode(X86::CMP64ri32); 3365 else 3366 Compare.setOpcode(X86::CMP64mi32); 3367 3368 // Original call address. 3369 for (unsigned i = 0; 3370 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3371 if (!CallInst.getOperand(i).isInst()) 3372 Compare.addOperand(CallInst.getOperand(i)); 3373 3374 // Target address. 3375 if (Targets[i].first || LoadElim) { 3376 const MCSymbol *Sym = 3377 LoadElim ? VtableSyms[i].first : Targets[i].first; 3378 const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0; 3379 const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx); 3380 if (Addend) 3381 Expr = MCBinaryExpr::createAdd( 3382 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx); 3383 Compare.addOperand(MCOperand::createExpr(Expr)); 3384 } else { 3385 const uint64_t Addr = Targets[i].second; 3386 // Immediate address is out of sign extended 32 bit range. 3387 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3388 return BlocksVectorTy(); 3389 3390 Compare.addOperand(MCOperand::createImm(Addr)); 3391 } 3392 } 3393 3394 // jump to next target compare. 3395 NextTarget = 3396 Ctx->createNamedTempSymbol(); // generate label for the next block 3397 NewCall->push_back(CallInst); 3398 3399 if (IsJumpTable) { 3400 MCInst &Je = NewCall->back(); 3401 3402 // Jump to next compare if target addresses don't match. 3403 Je.clear(); 3404 Je.setOpcode(X86::JCC_1); 3405 if (Targets[i].first) 3406 Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3407 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3408 else 3409 Je.addOperand(MCOperand::createImm(Targets[i].second)); 3410 3411 Je.addOperand(MCOperand::createImm(X86::COND_E)); 3412 assert(!isInvoke(CallInst)); 3413 } else { 3414 MCInst &Jne = NewCall->back(); 3415 3416 // Jump to next compare if target addresses don't match. 3417 Jne.clear(); 3418 Jne.setOpcode(X86::JCC_1); 3419 Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3420 NextTarget, MCSymbolRefExpr::VK_None, *Ctx))); 3421 Jne.addOperand(MCOperand::createImm(X86::COND_NE)); 3422 3423 // Call specific target directly. 3424 Results.emplace_back(Ctx->createNamedTempSymbol(), 3425 InstructionListType()); 3426 NewCall = &Results.back().second; 3427 NewCall->push_back(CallInst); 3428 MCInst &CallOrJmp = NewCall->back(); 3429 3430 CallOrJmp.clear(); 3431 3432 if (MinimizeCodeSize && !LoadElim) { 3433 CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r); 3434 CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg)); 3435 } else { 3436 CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 3437 3438 if (Targets[i].first) 3439 CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3440 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3441 else 3442 CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second)); 3443 } 3444 if (IsTailCall) 3445 setTailCall(CallOrJmp); 3446 3447 if (CallOrJmp.getOpcode() == X86::CALL64r || 3448 CallOrJmp.getOpcode() == X86::CALL64pcrel32) { 3449 if (std::optional<uint32_t> Offset = getOffset(CallInst)) 3450 // Annotated as duplicated call 3451 setOffset(CallOrJmp, *Offset); 3452 } 3453 3454 if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) { 3455 // Copy over any EH or GNU args size information from the original 3456 // call. 3457 std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst); 3458 if (EHInfo) 3459 addEHInfo(CallOrJmp, *EHInfo); 3460 int64_t GnuArgsSize = getGnuArgsSize(CallInst); 3461 if (GnuArgsSize >= 0) 3462 addGnuArgsSize(CallOrJmp, GnuArgsSize); 3463 } 3464 3465 if (!IsTailCall) { 3466 // The fallthrough block for the most common target should be 3467 // the merge block. 3468 if (i == 0) { 3469 // Fallthrough to merge block. 3470 MergeBlock = Ctx->createNamedTempSymbol(); 3471 } else { 3472 // Insert jump to the merge block if we are not doing a fallthrough. 3473 jumpToMergeBlock(*NewCall); 3474 } 3475 } 3476 } 3477 } 3478 3479 // Cold call block. 3480 Results.emplace_back(NextTarget, InstructionListType()); 3481 InstructionListType &NewCall = Results.back().second; 3482 for (const MCInst *Inst : MethodFetchInsns) 3483 if (Inst != &CallInst) 3484 NewCall.push_back(*Inst); 3485 NewCall.push_back(CallInst); 3486 3487 // Jump to merge block from cold call block 3488 if (!IsTailCall && !IsJumpTable) { 3489 jumpToMergeBlock(NewCall); 3490 3491 // Record merge block 3492 Results.emplace_back(MergeBlock, InstructionListType()); 3493 } 3494 3495 return Results; 3496 } 3497 3498 BlocksVectorTy jumpTablePromotion( 3499 const MCInst &IJmpInst, 3500 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3501 const std::vector<MCInst *> &TargetFetchInsns, 3502 MCContext *Ctx) const override { 3503 assert(getJumpTable(IJmpInst) != 0); 3504 uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg"); 3505 if (IndexReg == 0) 3506 return BlocksVectorTy(); 3507 3508 BlocksVectorTy Results; 3509 3510 // Label for the current code block. 3511 MCSymbol *NextTarget = nullptr; 3512 3513 for (unsigned int i = 0; i < Targets.size(); ++i) { 3514 Results.emplace_back(NextTarget, InstructionListType()); 3515 InstructionListType *CurBB = &Results.back().second; 3516 3517 // Compare current index to a specific index. 3518 CurBB->emplace_back(MCInst()); 3519 MCInst &CompareInst = CurBB->back(); 3520 CompareInst.setLoc(IJmpInst.getLoc()); 3521 CompareInst.setOpcode(X86::CMP64ri32); 3522 CompareInst.addOperand(MCOperand::createReg(IndexReg)); 3523 3524 const uint64_t CaseIdx = Targets[i].second; 3525 // Immediate address is out of sign extended 32 bit range. 3526 if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) 3527 return BlocksVectorTy(); 3528 3529 CompareInst.addOperand(MCOperand::createImm(CaseIdx)); 3530 shortenInstruction(CompareInst, *Ctx->getSubtargetInfo()); 3531 3532 // jump to next target compare. 3533 NextTarget = 3534 Ctx->createNamedTempSymbol(); // generate label for the next block 3535 CurBB->push_back(MCInst()); 3536 3537 MCInst &JEInst = CurBB->back(); 3538 JEInst.setLoc(IJmpInst.getLoc()); 3539 3540 // Jump to target if indices match 3541 JEInst.setOpcode(X86::JCC_1); 3542 JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3543 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3544 JEInst.addOperand(MCOperand::createImm(X86::COND_E)); 3545 } 3546 3547 // Cold call block. 3548 Results.emplace_back(NextTarget, InstructionListType()); 3549 InstructionListType &CurBB = Results.back().second; 3550 for (const MCInst *Inst : TargetFetchInsns) 3551 if (Inst != &IJmpInst) 3552 CurBB.push_back(*Inst); 3553 3554 CurBB.push_back(IJmpInst); 3555 3556 return Results; 3557 } 3558 3559 private: 3560 bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3561 MCContext *Ctx) const { 3562 Inst.setOpcode(X86::MOV64rm); 3563 Inst.addOperand(MCOperand::createReg(Reg)); 3564 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3565 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3566 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3567 Inst.addOperand(MCOperand::createExpr( 3568 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3569 *Ctx))); // Displacement 3570 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3571 3572 return true; 3573 } 3574 3575 bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3576 MCContext *Ctx) const { 3577 Inst.setOpcode(X86::LEA64r); 3578 Inst.addOperand(MCOperand::createReg(Reg)); 3579 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3580 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3581 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3582 Inst.addOperand(MCOperand::createExpr( 3583 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3584 *Ctx))); // Displacement 3585 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3586 return true; 3587 } 3588 }; 3589 3590 } // namespace 3591 3592 namespace llvm { 3593 namespace bolt { 3594 3595 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis, 3596 const MCInstrInfo *Info, 3597 const MCRegisterInfo *RegInfo) { 3598 return new X86MCPlusBuilder(Analysis, Info, RegInfo); 3599 } 3600 3601 } // namespace bolt 3602 } // namespace llvm 3603