1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides X86-specific MCPlus builder. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86EncodingOptimization.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "X86MCSymbolizer.h" 17 #include "bolt/Core/MCPlus.h" 18 #include "bolt/Core/MCPlusBuilder.h" 19 #include "llvm/BinaryFormat/ELF.h" 20 #include "llvm/MC/MCContext.h" 21 #include "llvm/MC/MCFixupKindInfo.h" 22 #include "llvm/MC/MCInst.h" 23 #include "llvm/MC/MCInstBuilder.h" 24 #include "llvm/MC/MCInstrInfo.h" 25 #include "llvm/MC/MCRegister.h" 26 #include "llvm/MC/MCRegisterInfo.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/DataExtractor.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/Errc.h" 31 #include "llvm/Support/ErrorHandling.h" 32 #include "llvm/Support/ErrorOr.h" 33 #include <set> 34 35 #define DEBUG_TYPE "mcplus" 36 37 using namespace llvm; 38 using namespace bolt; 39 40 namespace opts { 41 42 extern cl::OptionCategory BoltOptCategory; 43 44 static cl::opt<bool> X86StripRedundantAddressSize( 45 "x86-strip-redundant-address-size", 46 cl::desc("Remove redundant Address-Size override prefix"), cl::init(true), 47 cl::cat(BoltOptCategory)); 48 49 } // namespace opts 50 51 namespace { 52 53 bool isMOVSX64rm32(const MCInst &Inst) { 54 return Inst.getOpcode() == X86::MOVSX64rm32; 55 } 56 57 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; } 58 59 bool isADDri(const MCInst &Inst) { 60 return Inst.getOpcode() == X86::ADD64ri32 || 61 Inst.getOpcode() == X86::ADD64ri8; 62 } 63 64 #define GET_INSTRINFO_OPERAND_TYPES_ENUM 65 #define GET_INSTRINFO_OPERAND_TYPE 66 #define GET_INSTRINFO_MEM_OPERAND_SIZE 67 #include "X86GenInstrInfo.inc" 68 69 class X86MCPlusBuilder : public MCPlusBuilder { 70 public: 71 X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, 72 const MCRegisterInfo *RegInfo) 73 : MCPlusBuilder(Analysis, Info, RegInfo) {} 74 75 std::unique_ptr<MCSymbolizer> 76 createTargetSymbolizer(BinaryFunction &Function, 77 bool CreateNewSymbols) const override { 78 return std::make_unique<X86MCSymbolizer>(Function, CreateNewSymbols); 79 } 80 81 bool isBranch(const MCInst &Inst) const override { 82 return Analysis->isBranch(Inst) && !isTailCall(Inst); 83 } 84 85 bool isNoop(const MCInst &Inst) const override { 86 return X86::isNOP(Inst.getOpcode()); 87 } 88 89 unsigned getCondCode(const MCInst &Inst) const override { 90 unsigned Opcode = Inst.getOpcode(); 91 if (X86::isJCC(Opcode)) 92 return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm(); 93 return X86::COND_INVALID; 94 } 95 96 unsigned getInvertedCondCode(unsigned CC) const override { 97 switch (CC) { 98 default: return X86::COND_INVALID; 99 case X86::COND_E: return X86::COND_NE; 100 case X86::COND_NE: return X86::COND_E; 101 case X86::COND_L: return X86::COND_GE; 102 case X86::COND_LE: return X86::COND_G; 103 case X86::COND_G: return X86::COND_LE; 104 case X86::COND_GE: return X86::COND_L; 105 case X86::COND_B: return X86::COND_AE; 106 case X86::COND_BE: return X86::COND_A; 107 case X86::COND_A: return X86::COND_BE; 108 case X86::COND_AE: return X86::COND_B; 109 case X86::COND_S: return X86::COND_NS; 110 case X86::COND_NS: return X86::COND_S; 111 case X86::COND_P: return X86::COND_NP; 112 case X86::COND_NP: return X86::COND_P; 113 case X86::COND_O: return X86::COND_NO; 114 case X86::COND_NO: return X86::COND_O; 115 } 116 } 117 118 unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override { 119 enum DecodedCondCode : uint8_t { 120 DCC_EQUAL = 0x1, 121 DCC_GREATER = 0x2, 122 DCC_LESSER = 0x4, 123 DCC_GREATER_OR_LESSER = 0x6, 124 DCC_UNSIGNED = 0x8, 125 DCC_SIGNED = 0x10, 126 DCC_INVALID = 0x20, 127 }; 128 129 auto decodeCondCode = [&](unsigned CC) -> uint8_t { 130 switch (CC) { 131 default: return DCC_INVALID; 132 case X86::COND_E: return DCC_EQUAL; 133 case X86::COND_NE: return DCC_GREATER | DCC_LESSER; 134 case X86::COND_L: return DCC_LESSER | DCC_SIGNED; 135 case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED; 136 case X86::COND_G: return DCC_GREATER | DCC_SIGNED; 137 case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED; 138 case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED; 139 case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED; 140 case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED; 141 case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED; 142 } 143 }; 144 145 uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2); 146 147 if (DCC & DCC_INVALID) 148 return X86::COND_INVALID; 149 150 if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED) 151 return X86::COND_INVALID; 152 153 switch (DCC) { 154 default: return X86::COND_INVALID; 155 case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE; 156 case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE; 157 case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE; 158 case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE; 159 case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE; 160 case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE; 161 case DCC_GREATER | DCC_LESSER: return X86::COND_NE; 162 case DCC_EQUAL | DCC_SIGNED: return X86::COND_E; 163 case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E; 164 case DCC_EQUAL: return X86::COND_E; 165 case DCC_LESSER | DCC_SIGNED: return X86::COND_L; 166 case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B; 167 case DCC_GREATER | DCC_SIGNED: return X86::COND_G; 168 case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A; 169 } 170 } 171 172 bool isValidCondCode(unsigned CC) const override { 173 return (CC != X86::COND_INVALID); 174 } 175 176 bool isBreakpoint(const MCInst &Inst) const override { 177 return Inst.getOpcode() == X86::INT3; 178 } 179 180 bool isPrefix(const MCInst &Inst) const override { 181 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 182 return X86II::isPrefix(Desc.TSFlags); 183 } 184 185 bool isRep(const MCInst &Inst) const override { 186 return Inst.getFlags() == X86::IP_HAS_REPEAT; 187 } 188 189 bool deleteREPPrefix(MCInst &Inst) const override { 190 if (Inst.getFlags() == X86::IP_HAS_REPEAT) { 191 Inst.setFlags(0); 192 return true; 193 } 194 return false; 195 } 196 197 // FIXME: For compatibility with old LLVM only! 198 bool isTerminator(const MCInst &Inst) const override { 199 unsigned Opcode = Inst.getOpcode(); 200 return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) || 201 X86::isUD2(Opcode); 202 } 203 204 bool isIndirectCall(const MCInst &Inst) const override { 205 return isCall(Inst) && 206 ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); 207 } 208 209 bool isPop(const MCInst &Inst) const override { 210 return getPopSize(Inst) == 0 ? false : true; 211 } 212 213 bool isTerminateBranch(const MCInst &Inst) const override { 214 return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64; 215 } 216 217 int getPopSize(const MCInst &Inst) const override { 218 switch (Inst.getOpcode()) { 219 case X86::POP16r: 220 case X86::POP16rmm: 221 case X86::POP16rmr: 222 case X86::POPF16: 223 case X86::POPA16: 224 case X86::POPDS16: 225 case X86::POPES16: 226 case X86::POPFS16: 227 case X86::POPGS16: 228 case X86::POPSS16: 229 return 2; 230 case X86::POP32r: 231 case X86::POP32rmm: 232 case X86::POP32rmr: 233 case X86::POPA32: 234 case X86::POPDS32: 235 case X86::POPES32: 236 case X86::POPF32: 237 case X86::POPFS32: 238 case X86::POPGS32: 239 case X86::POPSS32: 240 return 4; 241 case X86::POP64r: 242 case X86::POP64rmm: 243 case X86::POP64rmr: 244 case X86::POPF64: 245 case X86::POPFS64: 246 case X86::POPGS64: 247 return 8; 248 } 249 return 0; 250 } 251 252 bool isPush(const MCInst &Inst) const override { 253 return getPushSize(Inst) == 0 ? false : true; 254 } 255 256 int getPushSize(const MCInst &Inst) const override { 257 switch (Inst.getOpcode()) { 258 case X86::PUSH16i8: 259 case X86::PUSH16r: 260 case X86::PUSH16rmm: 261 case X86::PUSH16rmr: 262 case X86::PUSHA16: 263 case X86::PUSHCS16: 264 case X86::PUSHDS16: 265 case X86::PUSHES16: 266 case X86::PUSHF16: 267 case X86::PUSHFS16: 268 case X86::PUSHGS16: 269 case X86::PUSHSS16: 270 case X86::PUSH16i: 271 return 2; 272 case X86::PUSH32i8: 273 case X86::PUSH32r: 274 case X86::PUSH32rmm: 275 case X86::PUSH32rmr: 276 case X86::PUSHA32: 277 case X86::PUSHCS32: 278 case X86::PUSHDS32: 279 case X86::PUSHES32: 280 case X86::PUSHF32: 281 case X86::PUSHFS32: 282 case X86::PUSHGS32: 283 case X86::PUSHSS32: 284 case X86::PUSH32i: 285 return 4; 286 case X86::PUSH64i32: 287 case X86::PUSH64i8: 288 case X86::PUSH64r: 289 case X86::PUSH64rmm: 290 case X86::PUSH64rmr: 291 case X86::PUSHF64: 292 case X86::PUSHFS64: 293 case X86::PUSHGS64: 294 return 8; 295 } 296 return 0; 297 } 298 299 bool isSUB(const MCInst &Inst) const override { 300 return X86::isSUB(Inst.getOpcode()); 301 } 302 303 bool isLEA64r(const MCInst &Inst) const override { 304 return Inst.getOpcode() == X86::LEA64r; 305 } 306 307 bool isLeave(const MCInst &Inst) const override { 308 return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64; 309 } 310 311 bool isMoveMem2Reg(const MCInst &Inst) const override { 312 switch (Inst.getOpcode()) { 313 case X86::MOV16rm: 314 case X86::MOV32rm: 315 case X86::MOV64rm: 316 return true; 317 } 318 return false; 319 } 320 321 bool isUnsupportedBranch(const MCInst &Inst) const override { 322 switch (Inst.getOpcode()) { 323 default: 324 return false; 325 case X86::LOOP: 326 case X86::LOOPE: 327 case X86::LOOPNE: 328 case X86::JECXZ: 329 case X86::JRCXZ: 330 return true; 331 } 332 } 333 334 bool isLoad(const MCInst &Inst) const override { 335 if (isPop(Inst)) 336 return true; 337 338 int MemOpNo = getMemoryOperandNo(Inst); 339 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 340 341 if (MemOpNo == -1) 342 return false; 343 344 return MCII.mayLoad(); 345 } 346 347 bool isStore(const MCInst &Inst) const override { 348 if (isPush(Inst)) 349 return true; 350 351 int MemOpNo = getMemoryOperandNo(Inst); 352 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 353 354 if (MemOpNo == -1) 355 return false; 356 357 return MCII.mayStore(); 358 } 359 360 bool isCleanRegXOR(const MCInst &Inst) const override { 361 switch (Inst.getOpcode()) { 362 case X86::XOR16rr: 363 case X86::XOR32rr: 364 case X86::XOR64rr: 365 break; 366 default: 367 return false; 368 } 369 return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg()); 370 } 371 372 bool isPacked(const MCInst &Inst) const override { 373 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 374 return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD; 375 } 376 377 bool shouldRecordCodeRelocation(uint64_t RelType) const override { 378 switch (RelType) { 379 case ELF::R_X86_64_8: 380 case ELF::R_X86_64_16: 381 case ELF::R_X86_64_32: 382 case ELF::R_X86_64_32S: 383 case ELF::R_X86_64_64: 384 case ELF::R_X86_64_PC8: 385 case ELF::R_X86_64_PC32: 386 case ELF::R_X86_64_PC64: 387 case ELF::R_X86_64_GOTPCRELX: 388 case ELF::R_X86_64_REX_GOTPCRELX: 389 return true; 390 case ELF::R_X86_64_PLT32: 391 case ELF::R_X86_64_GOTPCREL: 392 case ELF::R_X86_64_TPOFF32: 393 case ELF::R_X86_64_GOTTPOFF: 394 return false; 395 default: 396 llvm_unreachable("Unexpected x86 relocation type in code"); 397 } 398 } 399 400 unsigned getTrapFillValue() const override { return 0xCC; } 401 402 struct IndJmpMatcherFrag1 : MCInstMatcher { 403 std::unique_ptr<MCInstMatcher> Base; 404 std::unique_ptr<MCInstMatcher> Scale; 405 std::unique_ptr<MCInstMatcher> Index; 406 std::unique_ptr<MCInstMatcher> Offset; 407 408 IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 409 std::unique_ptr<MCInstMatcher> Scale, 410 std::unique_ptr<MCInstMatcher> Index, 411 std::unique_ptr<MCInstMatcher> Offset) 412 : Base(std::move(Base)), Scale(std::move(Scale)), 413 Index(std::move(Index)), Offset(std::move(Offset)) {} 414 415 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 416 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 417 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 418 return false; 419 420 if (CurInst->getOpcode() != X86::JMP64m) 421 return false; 422 423 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 424 if (MemOpNo == -1) 425 return false; 426 427 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 428 return false; 429 if (!Scale->match(MRI, MIB, this->InstrWindow, 430 MemOpNo + X86::AddrScaleAmt)) 431 return false; 432 if (!Index->match(MRI, MIB, this->InstrWindow, 433 MemOpNo + X86::AddrIndexReg)) 434 return false; 435 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 436 return false; 437 return true; 438 } 439 440 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 441 MIB.addAnnotation(*CurInst, Annotation, true); 442 Base->annotate(MIB, Annotation); 443 Scale->annotate(MIB, Annotation); 444 Index->annotate(MIB, Annotation); 445 Offset->annotate(MIB, Annotation); 446 } 447 }; 448 449 std::unique_ptr<MCInstMatcher> 450 matchIndJmp(std::unique_ptr<MCInstMatcher> Base, 451 std::unique_ptr<MCInstMatcher> Scale, 452 std::unique_ptr<MCInstMatcher> Index, 453 std::unique_ptr<MCInstMatcher> Offset) const override { 454 return std::unique_ptr<MCInstMatcher>( 455 new IndJmpMatcherFrag1(std::move(Base), std::move(Scale), 456 std::move(Index), std::move(Offset))); 457 } 458 459 struct IndJmpMatcherFrag2 : MCInstMatcher { 460 std::unique_ptr<MCInstMatcher> Reg; 461 462 IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg) 463 : Reg(std::move(Reg)) {} 464 465 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 466 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 467 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 468 return false; 469 470 if (CurInst->getOpcode() != X86::JMP64r) 471 return false; 472 473 return Reg->match(MRI, MIB, this->InstrWindow, 0); 474 } 475 476 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 477 MIB.addAnnotation(*CurInst, Annotation, true); 478 Reg->annotate(MIB, Annotation); 479 } 480 }; 481 482 std::unique_ptr<MCInstMatcher> 483 matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override { 484 return std::unique_ptr<MCInstMatcher>( 485 new IndJmpMatcherFrag2(std::move(Target))); 486 } 487 488 struct LoadMatcherFrag1 : MCInstMatcher { 489 std::unique_ptr<MCInstMatcher> Base; 490 std::unique_ptr<MCInstMatcher> Scale; 491 std::unique_ptr<MCInstMatcher> Index; 492 std::unique_ptr<MCInstMatcher> Offset; 493 494 LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 495 std::unique_ptr<MCInstMatcher> Scale, 496 std::unique_ptr<MCInstMatcher> Index, 497 std::unique_ptr<MCInstMatcher> Offset) 498 : Base(std::move(Base)), Scale(std::move(Scale)), 499 Index(std::move(Index)), Offset(std::move(Offset)) {} 500 501 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 502 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 503 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 504 return false; 505 506 if (CurInst->getOpcode() != X86::MOV64rm && 507 CurInst->getOpcode() != X86::MOVSX64rm32) 508 return false; 509 510 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 511 if (MemOpNo == -1) 512 return false; 513 514 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 515 return false; 516 if (!Scale->match(MRI, MIB, this->InstrWindow, 517 MemOpNo + X86::AddrScaleAmt)) 518 return false; 519 if (!Index->match(MRI, MIB, this->InstrWindow, 520 MemOpNo + X86::AddrIndexReg)) 521 return false; 522 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 523 return false; 524 return true; 525 } 526 527 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 528 MIB.addAnnotation(*CurInst, Annotation, true); 529 Base->annotate(MIB, Annotation); 530 Scale->annotate(MIB, Annotation); 531 Index->annotate(MIB, Annotation); 532 Offset->annotate(MIB, Annotation); 533 } 534 }; 535 536 std::unique_ptr<MCInstMatcher> 537 matchLoad(std::unique_ptr<MCInstMatcher> Base, 538 std::unique_ptr<MCInstMatcher> Scale, 539 std::unique_ptr<MCInstMatcher> Index, 540 std::unique_ptr<MCInstMatcher> Offset) const override { 541 return std::unique_ptr<MCInstMatcher>( 542 new LoadMatcherFrag1(std::move(Base), std::move(Scale), 543 std::move(Index), std::move(Offset))); 544 } 545 546 struct AddMatcher : MCInstMatcher { 547 std::unique_ptr<MCInstMatcher> A; 548 std::unique_ptr<MCInstMatcher> B; 549 550 AddMatcher(std::unique_ptr<MCInstMatcher> A, 551 std::unique_ptr<MCInstMatcher> B) 552 : A(std::move(A)), B(std::move(B)) {} 553 554 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 555 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 556 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 557 return false; 558 559 if (CurInst->getOpcode() == X86::ADD64rr || 560 CurInst->getOpcode() == X86::ADD64rr_DB || 561 CurInst->getOpcode() == X86::ADD64rr_REV) { 562 if (!A->match(MRI, MIB, this->InstrWindow, 1)) { 563 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 564 return false; 565 return A->match(MRI, MIB, this->InstrWindow, 2); 566 } 567 568 if (B->match(MRI, MIB, this->InstrWindow, 2)) 569 return true; 570 571 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 572 return false; 573 return A->match(MRI, MIB, this->InstrWindow, 2); 574 } 575 576 return false; 577 } 578 579 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 580 MIB.addAnnotation(*CurInst, Annotation, true); 581 A->annotate(MIB, Annotation); 582 B->annotate(MIB, Annotation); 583 } 584 }; 585 586 std::unique_ptr<MCInstMatcher> 587 matchAdd(std::unique_ptr<MCInstMatcher> A, 588 std::unique_ptr<MCInstMatcher> B) const override { 589 return std::unique_ptr<MCInstMatcher>( 590 new AddMatcher(std::move(A), std::move(B))); 591 } 592 593 struct LEAMatcher : MCInstMatcher { 594 std::unique_ptr<MCInstMatcher> Target; 595 596 LEAMatcher(std::unique_ptr<MCInstMatcher> Target) 597 : Target(std::move(Target)) {} 598 599 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 600 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 601 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 602 return false; 603 604 if (CurInst->getOpcode() != X86::LEA64r) 605 return false; 606 607 if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 || 608 CurInst->getOperand(1 + X86::AddrIndexReg).getReg() != 609 X86::NoRegister || 610 (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != 611 X86::NoRegister && 612 CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP)) 613 return false; 614 615 return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp); 616 } 617 618 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 619 MIB.addAnnotation(*CurInst, Annotation, true); 620 Target->annotate(MIB, Annotation); 621 } 622 }; 623 624 std::unique_ptr<MCInstMatcher> 625 matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override { 626 return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target))); 627 } 628 629 bool hasPCRelOperand(const MCInst &Inst) const override { 630 for (const MCOperand &Operand : Inst) 631 if (Operand.isReg() && Operand.getReg() == X86::RIP) 632 return true; 633 return false; 634 } 635 636 int getMemoryOperandNo(const MCInst &Inst) const override { 637 unsigned Opcode = Inst.getOpcode(); 638 const MCInstrDesc &Desc = Info->get(Opcode); 639 int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags); 640 if (MemOpNo >= 0) 641 MemOpNo += X86II::getOperandBias(Desc); 642 return MemOpNo; 643 } 644 645 bool hasEVEXEncoding(const MCInst &Inst) const override { 646 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 647 return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX; 648 } 649 650 bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override { 651 const auto *I = Insts.begin(); 652 while (I != Insts.end() && isPrefix(*I)) 653 ++I; 654 if (I == Insts.end()) 655 return false; 656 657 const MCInst &FirstInst = *I; 658 ++I; 659 while (I != Insts.end() && isPrefix(*I)) 660 ++I; 661 if (I == Insts.end()) 662 return false; 663 const MCInst &SecondInst = *I; 664 665 if (!isConditionalBranch(SecondInst)) 666 return false; 667 // Cannot fuse if the first instruction uses RIP-relative memory. 668 if (hasPCRelOperand(FirstInst)) 669 return false; 670 671 const X86::FirstMacroFusionInstKind CmpKind = 672 X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode()); 673 if (CmpKind == X86::FirstMacroFusionInstKind::Invalid) 674 return false; 675 676 X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst)); 677 X86::SecondMacroFusionInstKind BranchKind = 678 X86::classifySecondCondCodeInMacroFusion(CC); 679 if (BranchKind == X86::SecondMacroFusionInstKind::Invalid) 680 return false; 681 return X86::isMacroFused(CmpKind, BranchKind); 682 } 683 684 std::optional<X86MemOperand> 685 evaluateX86MemoryOperand(const MCInst &Inst) const override { 686 int MemOpNo = getMemoryOperandNo(Inst); 687 if (MemOpNo < 0) 688 return std::nullopt; 689 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 690 691 if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst)) 692 return std::nullopt; 693 694 const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg); 695 const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt); 696 const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg); 697 const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp); 698 const MCOperand &Segment = 699 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg); 700 701 // Make sure it is a well-formed memory operand. 702 if (!Base.isReg() || !Scale.isImm() || !Index.isReg() || 703 (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg()) 704 return std::nullopt; 705 706 X86MemOperand MO; 707 MO.BaseRegNum = Base.getReg(); 708 MO.ScaleImm = Scale.getImm(); 709 MO.IndexRegNum = Index.getReg(); 710 MO.DispImm = Disp.isImm() ? Disp.getImm() : 0; 711 MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr; 712 MO.SegRegNum = Segment.getReg(); 713 return MO; 714 } 715 716 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, 717 uint64_t Address, 718 uint64_t Size) const override { 719 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 720 if (!MO) 721 return false; 722 723 // Make sure it's a well-formed addressing we can statically evaluate. 724 if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) || 725 MO->IndexRegNum != X86::NoRegister || 726 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 727 return false; 728 729 Target = MO->DispImm; 730 if (MO->BaseRegNum == X86::RIP) { 731 assert(Size != 0 && "instruction size required in order to statically " 732 "evaluate RIP-relative address"); 733 Target += Address + Size; 734 } 735 return true; 736 } 737 738 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { 739 int MemOpNo = getMemoryOperandNo(Inst); 740 if (MemOpNo < 0) 741 return Inst.end(); 742 return Inst.begin() + (MemOpNo + X86::AddrDisp); 743 } 744 745 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { 746 MCOperand *OI = getMemOperandDisp(Inst); 747 if (OI == Inst.end()) 748 return false; 749 *OI = Operand; 750 return true; 751 } 752 753 /// Get the registers used as function parameters. 754 /// This function is specific to the x86_64 abi on Linux. 755 BitVector getRegsUsedAsParams() const override { 756 BitVector Regs = BitVector(RegInfo->getNumRegs(), false); 757 Regs |= getAliases(X86::RSI); 758 Regs |= getAliases(X86::RDI); 759 Regs |= getAliases(X86::RDX); 760 Regs |= getAliases(X86::RCX); 761 Regs |= getAliases(X86::R8); 762 Regs |= getAliases(X86::R9); 763 return Regs; 764 } 765 766 void getCalleeSavedRegs(BitVector &Regs) const override { 767 Regs |= getAliases(X86::RBX); 768 Regs |= getAliases(X86::RBP); 769 Regs |= getAliases(X86::R12); 770 Regs |= getAliases(X86::R13); 771 Regs |= getAliases(X86::R14); 772 Regs |= getAliases(X86::R15); 773 } 774 775 void getDefaultDefIn(BitVector &Regs) const override { 776 assert(Regs.size() >= RegInfo->getNumRegs() && 777 "The size of BitVector is less than RegInfo->getNumRegs()."); 778 Regs.set(X86::RAX); 779 Regs.set(X86::RCX); 780 Regs.set(X86::RDX); 781 Regs.set(X86::RSI); 782 Regs.set(X86::RDI); 783 Regs.set(X86::R8); 784 Regs.set(X86::R9); 785 Regs.set(X86::XMM0); 786 Regs.set(X86::XMM1); 787 Regs.set(X86::XMM2); 788 Regs.set(X86::XMM3); 789 Regs.set(X86::XMM4); 790 Regs.set(X86::XMM5); 791 Regs.set(X86::XMM6); 792 Regs.set(X86::XMM7); 793 } 794 795 void getDefaultLiveOut(BitVector &Regs) const override { 796 assert(Regs.size() >= RegInfo->getNumRegs() && 797 "The size of BitVector is less than RegInfo->getNumRegs()."); 798 Regs |= getAliases(X86::RAX); 799 Regs |= getAliases(X86::RDX); 800 Regs |= getAliases(X86::RCX); 801 Regs |= getAliases(X86::XMM0); 802 Regs |= getAliases(X86::XMM1); 803 } 804 805 void getGPRegs(BitVector &Regs, bool IncludeAlias) const override { 806 if (IncludeAlias) { 807 Regs |= getAliases(X86::RAX); 808 Regs |= getAliases(X86::RBX); 809 Regs |= getAliases(X86::RBP); 810 Regs |= getAliases(X86::RSI); 811 Regs |= getAliases(X86::RDI); 812 Regs |= getAliases(X86::RDX); 813 Regs |= getAliases(X86::RCX); 814 Regs |= getAliases(X86::R8); 815 Regs |= getAliases(X86::R9); 816 Regs |= getAliases(X86::R10); 817 Regs |= getAliases(X86::R11); 818 Regs |= getAliases(X86::R12); 819 Regs |= getAliases(X86::R13); 820 Regs |= getAliases(X86::R14); 821 Regs |= getAliases(X86::R15); 822 return; 823 } 824 Regs.set(X86::RAX); 825 Regs.set(X86::RBX); 826 Regs.set(X86::RBP); 827 Regs.set(X86::RSI); 828 Regs.set(X86::RDI); 829 Regs.set(X86::RDX); 830 Regs.set(X86::RCX); 831 Regs.set(X86::R8); 832 Regs.set(X86::R9); 833 Regs.set(X86::R10); 834 Regs.set(X86::R11); 835 Regs.set(X86::R12); 836 Regs.set(X86::R13); 837 Regs.set(X86::R14); 838 Regs.set(X86::R15); 839 } 840 841 void getClassicGPRegs(BitVector &Regs) const override { 842 Regs |= getAliases(X86::RAX); 843 Regs |= getAliases(X86::RBX); 844 Regs |= getAliases(X86::RBP); 845 Regs |= getAliases(X86::RSI); 846 Regs |= getAliases(X86::RDI); 847 Regs |= getAliases(X86::RDX); 848 Regs |= getAliases(X86::RCX); 849 } 850 851 void getRepRegs(BitVector &Regs) const override { 852 Regs |= getAliases(X86::RCX); 853 } 854 855 MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override { 856 Reg = getX86SubSuperRegister(Reg, Size * 8); 857 assert((Reg != X86::NoRegister) && "Invalid register"); 858 return Reg; 859 } 860 861 bool isUpper8BitReg(MCPhysReg Reg) const override { 862 switch (Reg) { 863 case X86::AH: 864 case X86::BH: 865 case X86::CH: 866 case X86::DH: 867 return true; 868 default: 869 return false; 870 } 871 } 872 873 bool cannotUseREX(const MCInst &Inst) const override { 874 switch (Inst.getOpcode()) { 875 case X86::MOV8mr_NOREX: 876 case X86::MOV8rm_NOREX: 877 case X86::MOV8rr_NOREX: 878 case X86::MOVSX32rm8_NOREX: 879 case X86::MOVSX32rr8_NOREX: 880 case X86::MOVZX32rm8_NOREX: 881 case X86::MOVZX32rr8_NOREX: 882 case X86::MOV8mr: 883 case X86::MOV8rm: 884 case X86::MOV8rr: 885 case X86::MOVSX32rm8: 886 case X86::MOVSX32rr8: 887 case X86::MOVZX32rm8: 888 case X86::MOVZX32rr8: 889 case X86::TEST8ri: 890 for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) { 891 if (!Operand.isReg()) 892 continue; 893 if (isUpper8BitReg(Operand.getReg())) 894 return true; 895 } 896 [[fallthrough]]; 897 default: 898 return false; 899 } 900 } 901 902 static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) { 903 using namespace llvm::X86; 904 int OpType = getOperandType(Inst.getOpcode(), MemOpNo); 905 return getMemOperandSize(OpType) / 8; 906 } 907 908 /// Classifying a stack access as *not* "SIMPLE" here means we don't know how 909 /// to change this instruction memory access. It will disable any changes to 910 /// the stack layout, so we can't do the most aggressive form of shrink 911 /// wrapping. We must do so in a way that keeps the original stack layout. 912 /// Otherwise you need to adjust the offset of all instructions accessing the 913 /// stack: we can't do that anymore because there is one instruction that is 914 /// not simple. There are other implications as well. We have heuristics to 915 /// detect when a register is callee-saved and thus eligible for shrink 916 /// wrapping. If you are restoring a register using a non-simple stack access, 917 /// then it is classified as NOT callee-saved, and it disables shrink wrapping 918 /// for *that* register (but not for others). 919 /// 920 /// Classifying a stack access as "size 0" or detecting an indexed memory 921 /// access (to address a vector, for example) here means we know there is a 922 /// stack access, but we can't quite understand how wide is the access in 923 /// bytes. This is very serious because we can't understand how memory 924 /// accesses alias with each other for this function. This will essentially 925 /// disable not only shrink wrapping but all frame analysis, it will fail it 926 /// as "we don't understand this function and we give up on it". 927 bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, 928 bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm, 929 uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size, 930 bool &IsSimple, bool &IsIndexed) const override { 931 // Detect simple push/pop cases first 932 if (int Sz = getPushSize(Inst)) { 933 IsLoad = false; 934 IsStore = true; 935 IsStoreFromReg = true; 936 StackPtrReg = X86::RSP; 937 StackOffset = -Sz; 938 Size = Sz; 939 IsSimple = true; 940 if (Inst.getOperand(0).isImm()) 941 SrcImm = Inst.getOperand(0).getImm(); 942 else if (Inst.getOperand(0).isReg()) 943 Reg = Inst.getOperand(0).getReg(); 944 else 945 IsSimple = false; 946 947 return true; 948 } 949 if (int Sz = getPopSize(Inst)) { 950 IsLoad = true; 951 IsStore = false; 952 if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) { 953 IsSimple = false; 954 } else { 955 Reg = Inst.getOperand(0).getReg(); 956 IsSimple = true; 957 } 958 StackPtrReg = X86::RSP; 959 StackOffset = 0; 960 Size = Sz; 961 return true; 962 } 963 964 struct InstInfo { 965 // Size in bytes that Inst loads from memory. 966 uint8_t DataSize; 967 bool IsLoad; 968 bool IsStore; 969 bool StoreFromReg; 970 bool Simple; 971 }; 972 973 InstInfo I; 974 int MemOpNo = getMemoryOperandNo(Inst); 975 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 976 // If it is not dealing with a memory operand, we discard it 977 if (MemOpNo == -1 || MCII.isCall()) 978 return false; 979 980 switch (Inst.getOpcode()) { 981 default: { 982 bool IsLoad = MCII.mayLoad(); 983 bool IsStore = MCII.mayStore(); 984 // Is it LEA? (deals with memory but is not loading nor storing) 985 if (!IsLoad && !IsStore) { 986 I = {0, IsLoad, IsStore, false, false}; 987 break; 988 } 989 uint8_t Sz = getMemDataSize(Inst, MemOpNo); 990 I = {Sz, IsLoad, IsStore, false, false}; 991 break; 992 } 993 // Report simple stack accesses 994 case X86::MOV8rm: I = {1, true, false, false, true}; break; 995 case X86::MOV16rm: I = {2, true, false, false, true}; break; 996 case X86::MOV32rm: I = {4, true, false, false, true}; break; 997 case X86::MOV64rm: I = {8, true, false, false, true}; break; 998 case X86::MOV8mr: I = {1, false, true, true, true}; break; 999 case X86::MOV16mr: I = {2, false, true, true, true}; break; 1000 case X86::MOV32mr: I = {4, false, true, true, true}; break; 1001 case X86::MOV64mr: I = {8, false, true, true, true}; break; 1002 case X86::MOV8mi: I = {1, false, true, false, true}; break; 1003 case X86::MOV16mi: I = {2, false, true, false, true}; break; 1004 case X86::MOV32mi: I = {4, false, true, false, true}; break; 1005 } // end switch (Inst.getOpcode()) 1006 1007 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1008 if (!MO) { 1009 LLVM_DEBUG(dbgs() << "Evaluate failed on "); 1010 LLVM_DEBUG(Inst.dump()); 1011 return false; 1012 } 1013 1014 // Make sure it's a stack access 1015 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) 1016 return false; 1017 1018 IsLoad = I.IsLoad; 1019 IsStore = I.IsStore; 1020 IsStoreFromReg = I.StoreFromReg; 1021 Size = I.DataSize; 1022 IsSimple = I.Simple; 1023 StackPtrReg = MO->BaseRegNum; 1024 StackOffset = MO->DispImm; 1025 IsIndexed = 1026 MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister; 1027 1028 if (!I.Simple) 1029 return true; 1030 1031 // Retrieve related register in simple MOV from/to stack operations. 1032 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 1033 if (I.IsLoad) { 1034 MCOperand RegOpnd = Inst.getOperand(0); 1035 assert(RegOpnd.isReg() && "unexpected destination operand"); 1036 Reg = RegOpnd.getReg(); 1037 } else if (I.IsStore) { 1038 MCOperand SrcOpnd = 1039 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1040 if (I.StoreFromReg) { 1041 assert(SrcOpnd.isReg() && "unexpected source operand"); 1042 Reg = SrcOpnd.getReg(); 1043 } else { 1044 assert(SrcOpnd.isImm() && "unexpected source operand"); 1045 SrcImm = SrcOpnd.getImm(); 1046 } 1047 } 1048 1049 return true; 1050 } 1051 1052 void changeToPushOrPop(MCInst &Inst) const override { 1053 assert(!isPush(Inst) && !isPop(Inst)); 1054 1055 struct InstInfo { 1056 // Size in bytes that Inst loads from memory. 1057 uint8_t DataSize; 1058 bool IsLoad; 1059 bool StoreFromReg; 1060 }; 1061 1062 InstInfo I; 1063 switch (Inst.getOpcode()) { 1064 default: { 1065 llvm_unreachable("Unhandled opcode"); 1066 return; 1067 } 1068 case X86::MOV16rm: I = {2, true, false}; break; 1069 case X86::MOV32rm: I = {4, true, false}; break; 1070 case X86::MOV64rm: I = {8, true, false}; break; 1071 case X86::MOV16mr: I = {2, false, true}; break; 1072 case X86::MOV32mr: I = {4, false, true}; break; 1073 case X86::MOV64mr: I = {8, false, true}; break; 1074 case X86::MOV16mi: I = {2, false, false}; break; 1075 case X86::MOV32mi: I = {4, false, false}; break; 1076 } // end switch (Inst.getOpcode()) 1077 1078 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1079 if (!MO) { 1080 llvm_unreachable("Evaluate failed"); 1081 return; 1082 } 1083 // Make sure it's a stack access 1084 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) { 1085 llvm_unreachable("Not a stack access"); 1086 return; 1087 } 1088 1089 unsigned MemOpOffset = getMemoryOperandNo(Inst); 1090 unsigned NewOpcode = 0; 1091 if (I.IsLoad) { 1092 switch (I.DataSize) { 1093 case 2: NewOpcode = X86::POP16r; break; 1094 case 4: NewOpcode = X86::POP32r; break; 1095 case 8: NewOpcode = X86::POP64r; break; 1096 default: 1097 llvm_unreachable("Unexpected size"); 1098 } 1099 unsigned RegOpndNum = Inst.getOperand(0).getReg(); 1100 Inst.clear(); 1101 Inst.setOpcode(NewOpcode); 1102 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1103 } else { 1104 MCOperand SrcOpnd = 1105 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1106 if (I.StoreFromReg) { 1107 switch (I.DataSize) { 1108 case 2: NewOpcode = X86::PUSH16r; break; 1109 case 4: NewOpcode = X86::PUSH32r; break; 1110 case 8: NewOpcode = X86::PUSH64r; break; 1111 default: 1112 llvm_unreachable("Unexpected size"); 1113 } 1114 assert(SrcOpnd.isReg() && "Unexpected source operand"); 1115 unsigned RegOpndNum = SrcOpnd.getReg(); 1116 Inst.clear(); 1117 Inst.setOpcode(NewOpcode); 1118 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1119 } else { 1120 switch (I.DataSize) { 1121 case 2: NewOpcode = X86::PUSH16i8; break; 1122 case 4: NewOpcode = X86::PUSH32i8; break; 1123 case 8: NewOpcode = X86::PUSH64i32; break; 1124 default: 1125 llvm_unreachable("Unexpected size"); 1126 } 1127 assert(SrcOpnd.isImm() && "Unexpected source operand"); 1128 int64_t SrcImm = SrcOpnd.getImm(); 1129 Inst.clear(); 1130 Inst.setOpcode(NewOpcode); 1131 Inst.addOperand(MCOperand::createImm(SrcImm)); 1132 } 1133 } 1134 } 1135 1136 bool isStackAdjustment(const MCInst &Inst) const override { 1137 switch (Inst.getOpcode()) { 1138 default: 1139 return false; 1140 case X86::SUB64ri32: 1141 case X86::SUB64ri8: 1142 case X86::ADD64ri32: 1143 case X86::ADD64ri8: 1144 case X86::LEA64r: 1145 break; 1146 } 1147 1148 return any_of(defOperands(Inst), [](const MCOperand &Op) { 1149 return Op.isReg() && Op.getReg() == X86::RSP; 1150 }); 1151 } 1152 1153 bool 1154 evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output, 1155 std::pair<MCPhysReg, int64_t> Input1, 1156 std::pair<MCPhysReg, int64_t> Input2) const override { 1157 1158 auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> { 1159 if (Reg == Input1.first) 1160 return Input1.second; 1161 if (Reg == Input2.first) 1162 return Input2.second; 1163 return make_error_code(errc::result_out_of_range); 1164 }; 1165 1166 switch (Inst.getOpcode()) { 1167 default: 1168 return false; 1169 1170 case X86::SUB64ri32: 1171 case X86::SUB64ri8: 1172 if (!Inst.getOperand(2).isImm()) 1173 return false; 1174 if (ErrorOr<int64_t> InputVal = 1175 getOperandVal(Inst.getOperand(1).getReg())) 1176 Output = *InputVal - Inst.getOperand(2).getImm(); 1177 else 1178 return false; 1179 break; 1180 case X86::ADD64ri32: 1181 case X86::ADD64ri8: 1182 if (!Inst.getOperand(2).isImm()) 1183 return false; 1184 if (ErrorOr<int64_t> InputVal = 1185 getOperandVal(Inst.getOperand(1).getReg())) 1186 Output = *InputVal + Inst.getOperand(2).getImm(); 1187 else 1188 return false; 1189 break; 1190 case X86::ADD64i32: 1191 if (!Inst.getOperand(0).isImm()) 1192 return false; 1193 if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX)) 1194 Output = *InputVal + Inst.getOperand(0).getImm(); 1195 else 1196 return false; 1197 break; 1198 1199 case X86::LEA64r: { 1200 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1201 if (!MO) 1202 return false; 1203 1204 if (MO->BaseRegNum == X86::NoRegister || 1205 MO->IndexRegNum != X86::NoRegister || 1206 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 1207 return false; 1208 1209 if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum)) 1210 Output = *InputVal + MO->DispImm; 1211 else 1212 return false; 1213 1214 break; 1215 } 1216 } 1217 return true; 1218 } 1219 1220 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, 1221 MCPhysReg &To) const override { 1222 switch (Inst.getOpcode()) { 1223 default: 1224 return false; 1225 case X86::LEAVE: 1226 case X86::LEAVE64: 1227 To = getStackPointer(); 1228 From = getFramePointer(); 1229 return true; 1230 case X86::MOV64rr: 1231 To = Inst.getOperand(0).getReg(); 1232 From = Inst.getOperand(1).getReg(); 1233 return true; 1234 } 1235 } 1236 1237 MCPhysReg getStackPointer() const override { return X86::RSP; } 1238 MCPhysReg getFramePointer() const override { return X86::RBP; } 1239 MCPhysReg getFlagsReg() const override { return X86::EFLAGS; } 1240 1241 bool escapesVariable(const MCInst &Inst, 1242 bool HasFramePointer) const override { 1243 int MemOpNo = getMemoryOperandNo(Inst); 1244 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 1245 const unsigned NumDefs = MCII.getNumDefs(); 1246 static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |= 1247 getAliases(X86::RBP)); 1248 static BitVector SPAliases(getAliases(X86::RSP)); 1249 1250 // FIXME: PUSH can be technically a leak, but let's ignore this for now 1251 // because a lot of harmless prologue code will spill SP to the stack. 1252 // Unless push is clearly pushing an object address to the stack as 1253 // demonstrated by having a MemOp. 1254 bool IsPush = isPush(Inst); 1255 if (IsPush && MemOpNo == -1) 1256 return false; 1257 1258 // We use this to detect LEA (has memop but does not access mem) 1259 bool AccessMem = MCII.mayLoad() || MCII.mayStore(); 1260 bool DoesLeak = false; 1261 for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) { 1262 // Ignore if SP/BP is used to dereference memory -- that's fine 1263 if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo && 1264 I <= MemOpNo + 5) 1265 continue; 1266 // Ignore if someone is writing to SP/BP 1267 if (I < static_cast<int>(NumDefs)) 1268 continue; 1269 1270 const MCOperand &Operand = Inst.getOperand(I); 1271 if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) { 1272 DoesLeak = true; 1273 break; 1274 } 1275 if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) { 1276 DoesLeak = true; 1277 break; 1278 } 1279 } 1280 1281 // If potential leak, check if it is not just writing to itself/sp/bp 1282 if (DoesLeak) { 1283 DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) { 1284 assert(Operand.isReg()); 1285 MCPhysReg Reg = Operand.getReg(); 1286 return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg]; 1287 }); 1288 } 1289 return DoesLeak; 1290 } 1291 1292 bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override { 1293 unsigned ImmOpNo = -1U; 1294 int MemOpNo = getMemoryOperandNo(Inst); 1295 if (MemOpNo != -1) 1296 ImmOpNo = MemOpNo + X86::AddrDisp; 1297 else 1298 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 1299 ++Index) 1300 if (Inst.getOperand(Index).isImm()) 1301 ImmOpNo = Index; 1302 if (ImmOpNo == -1U) 1303 return false; 1304 1305 MCOperand &Operand = Inst.getOperand(ImmOpNo); 1306 Amt += Operand.getImm(); 1307 Operand.setImm(Amt); 1308 // Check for the need for relaxation 1309 if (int64_t(Amt) == int64_t(int8_t(Amt))) 1310 return true; 1311 1312 // Relax instruction 1313 switch (Inst.getOpcode()) { 1314 case X86::SUB64ri8: 1315 Inst.setOpcode(X86::SUB64ri32); 1316 break; 1317 case X86::ADD64ri8: 1318 Inst.setOpcode(X86::ADD64ri32); 1319 break; 1320 default: 1321 // No need for relaxation 1322 break; 1323 } 1324 return true; 1325 } 1326 1327 /// TODO: this implementation currently works for the most common opcodes that 1328 /// load from memory. It can be extended to work with memory store opcodes as 1329 /// well as more memory load opcodes. 1330 bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, 1331 uint64_t Offset) const override { 1332 enum CheckSignExt : uint8_t { 1333 NOCHECK = 0, 1334 CHECK8, 1335 CHECK32, 1336 }; 1337 1338 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 1339 struct InstInfo { 1340 // Size in bytes that Inst loads from memory. 1341 uint8_t DataSize; 1342 1343 // True when the target operand has to be duplicated because the opcode 1344 // expects a LHS operand. 1345 bool HasLHS; 1346 1347 // List of checks and corresponding opcodes to be used. We try to use the 1348 // smallest possible immediate value when various sizes are available, 1349 // hence we may need to check whether a larger constant fits in a smaller 1350 // immediate. 1351 CheckList Checks; 1352 }; 1353 1354 InstInfo I; 1355 1356 switch (Inst.getOpcode()) { 1357 default: { 1358 switch (getPopSize(Inst)) { 1359 case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1360 case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1361 case 8: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1362 {NOCHECK, X86::MOV64rm}}}; break; 1363 default: return false; 1364 } 1365 break; 1366 } 1367 1368 // MOV 1369 case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 1370 case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1371 case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1372 case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1373 {NOCHECK, X86::MOV64rm}}}; break; 1374 1375 // MOVZX 1376 case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 1377 case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 1378 case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 1379 1380 // CMP 1381 case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 1382 case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8}, 1383 {NOCHECK, X86::CMP16ri}}}; break; 1384 case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8}, 1385 {NOCHECK, X86::CMP32ri}}}; break; 1386 case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8}, 1387 {CHECK32, X86::CMP64ri32}, 1388 {NOCHECK, X86::CMP64rm}}}; break; 1389 1390 // TEST 1391 case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 1392 case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 1393 case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 1394 case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 1395 {NOCHECK, X86::TEST64mr}}}; break; 1396 1397 // ADD 1398 case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 1399 case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8}, 1400 {NOCHECK, X86::ADD16ri}}}; break; 1401 case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8}, 1402 {NOCHECK, X86::ADD32ri}}}; break; 1403 case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8}, 1404 {CHECK32, X86::ADD64ri32}, 1405 {NOCHECK, X86::ADD64rm}}}; break; 1406 1407 // SUB 1408 case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 1409 case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8}, 1410 {NOCHECK, X86::SUB16ri}}}; break; 1411 case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8}, 1412 {NOCHECK, X86::SUB32ri}}}; break; 1413 case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8}, 1414 {CHECK32, X86::SUB64ri32}, 1415 {NOCHECK, X86::SUB64rm}}}; break; 1416 1417 // AND 1418 case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 1419 case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8}, 1420 {NOCHECK, X86::AND16ri}}}; break; 1421 case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8}, 1422 {NOCHECK, X86::AND32ri}}}; break; 1423 case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8}, 1424 {CHECK32, X86::AND64ri32}, 1425 {NOCHECK, X86::AND64rm}}}; break; 1426 1427 // OR 1428 case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 1429 case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8}, 1430 {NOCHECK, X86::OR16ri}}}; break; 1431 case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8}, 1432 {NOCHECK, X86::OR32ri}}}; break; 1433 case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8}, 1434 {CHECK32, X86::OR64ri32}, 1435 {NOCHECK, X86::OR64rm}}}; break; 1436 1437 // XOR 1438 case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 1439 case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8}, 1440 {NOCHECK, X86::XOR16ri}}}; break; 1441 case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8}, 1442 {NOCHECK, X86::XOR32ri}}}; break; 1443 case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8}, 1444 {CHECK32, X86::XOR64ri32}, 1445 {NOCHECK, X86::XOR64rm}}}; break; 1446 } 1447 1448 // Compute the immediate value. 1449 assert(Offset + I.DataSize <= ConstantData.size() && 1450 "invalid offset for given constant data"); 1451 int64_t ImmVal = 1452 DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize); 1453 1454 // Compute the new opcode. 1455 unsigned NewOpcode = 0; 1456 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 1457 NewOpcode = Check.second; 1458 if (Check.first == NOCHECK) 1459 break; 1460 if (Check.first == CHECK8 && isInt<8>(ImmVal)) 1461 break; 1462 if (Check.first == CHECK32 && isInt<32>(ImmVal)) 1463 break; 1464 } 1465 if (NewOpcode == Inst.getOpcode()) 1466 return false; 1467 1468 // Modify the instruction. 1469 MCOperand ImmOp = MCOperand::createImm(ImmVal); 1470 uint32_t TargetOpNum = 0; 1471 // Test instruction does not follow the regular pattern of putting the 1472 // memory reference of a load (5 MCOperands) last in the list of operands. 1473 // Since it is not modifying the register operand, it is not treated as 1474 // a destination operand and it is not the first operand as it is in the 1475 // other instructions we treat here. 1476 if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri || 1477 NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32) 1478 TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands; 1479 1480 MCOperand TargetOp = Inst.getOperand(TargetOpNum); 1481 Inst.clear(); 1482 Inst.setOpcode(NewOpcode); 1483 Inst.addOperand(TargetOp); 1484 if (I.HasLHS) 1485 Inst.addOperand(TargetOp); 1486 Inst.addOperand(ImmOp); 1487 1488 return true; 1489 } 1490 1491 /// TODO: this implementation currently works for the most common opcodes that 1492 /// load from memory. It can be extended to work with memory store opcodes as 1493 /// well as more memory load opcodes. 1494 bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override { 1495 unsigned NewOpcode; 1496 1497 switch (Inst.getOpcode()) { 1498 default: { 1499 switch (getPopSize(Inst)) { 1500 case 2: NewOpcode = X86::MOV16rr; break; 1501 case 4: NewOpcode = X86::MOV32rr; break; 1502 case 8: NewOpcode = X86::MOV64rr; break; 1503 default: return false; 1504 } 1505 break; 1506 } 1507 1508 // MOV 1509 case X86::MOV8rm: NewOpcode = X86::MOV8rr; break; 1510 case X86::MOV16rm: NewOpcode = X86::MOV16rr; break; 1511 case X86::MOV32rm: NewOpcode = X86::MOV32rr; break; 1512 case X86::MOV64rm: NewOpcode = X86::MOV64rr; break; 1513 } 1514 1515 // Modify the instruction. 1516 MCOperand RegOp = MCOperand::createReg(RegNum); 1517 MCOperand TargetOp = Inst.getOperand(0); 1518 Inst.clear(); 1519 Inst.setOpcode(NewOpcode); 1520 Inst.addOperand(TargetOp); 1521 Inst.addOperand(RegOp); 1522 1523 return true; 1524 } 1525 1526 bool isRedundantMove(const MCInst &Inst) const override { 1527 switch (Inst.getOpcode()) { 1528 default: 1529 return false; 1530 1531 // MOV 1532 case X86::MOV8rr: 1533 case X86::MOV16rr: 1534 case X86::MOV32rr: 1535 case X86::MOV64rr: 1536 break; 1537 } 1538 1539 assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg()); 1540 return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg(); 1541 } 1542 1543 bool requiresAlignedAddress(const MCInst &Inst) const override { 1544 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 1545 for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) { 1546 const MCOperandInfo &Op = Desc.operands()[I]; 1547 if (Op.OperandType != MCOI::OPERAND_REGISTER) 1548 continue; 1549 if (Op.RegClass == X86::VR128RegClassID) 1550 return true; 1551 } 1552 return false; 1553 } 1554 1555 bool convertJmpToTailCall(MCInst &Inst) override { 1556 if (isTailCall(Inst)) 1557 return false; 1558 1559 int NewOpcode; 1560 switch (Inst.getOpcode()) { 1561 default: 1562 return false; 1563 case X86::JMP_1: 1564 case X86::JMP_2: 1565 case X86::JMP_4: 1566 NewOpcode = X86::JMP_4; 1567 break; 1568 case X86::JMP16m: 1569 case X86::JMP32m: 1570 case X86::JMP64m: 1571 NewOpcode = X86::JMP32m; 1572 break; 1573 case X86::JMP16r: 1574 case X86::JMP32r: 1575 case X86::JMP64r: 1576 NewOpcode = X86::JMP32r; 1577 break; 1578 } 1579 1580 Inst.setOpcode(NewOpcode); 1581 setTailCall(Inst); 1582 return true; 1583 } 1584 1585 bool convertTailCallToJmp(MCInst &Inst) override { 1586 int NewOpcode; 1587 switch (Inst.getOpcode()) { 1588 default: 1589 return false; 1590 case X86::JMP_4: 1591 NewOpcode = X86::JMP_1; 1592 break; 1593 case X86::JMP32m: 1594 NewOpcode = X86::JMP64m; 1595 break; 1596 case X86::JMP32r: 1597 NewOpcode = X86::JMP64r; 1598 break; 1599 } 1600 1601 Inst.setOpcode(NewOpcode); 1602 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1603 clearOffset(Inst); 1604 return true; 1605 } 1606 1607 bool convertTailCallToCall(MCInst &Inst) override { 1608 int NewOpcode; 1609 switch (Inst.getOpcode()) { 1610 default: 1611 return false; 1612 case X86::JMP_4: 1613 NewOpcode = X86::CALL64pcrel32; 1614 break; 1615 case X86::JMP32m: 1616 NewOpcode = X86::CALL64m; 1617 break; 1618 case X86::JMP32r: 1619 NewOpcode = X86::CALL64r; 1620 break; 1621 } 1622 1623 Inst.setOpcode(NewOpcode); 1624 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1625 return true; 1626 } 1627 1628 bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation, 1629 MCContext *Ctx) override { 1630 assert((Inst.getOpcode() == X86::CALL64pcrel32 || 1631 (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) && 1632 "64-bit direct (tail) call instruction expected"); 1633 const auto NewOpcode = 1634 (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m; 1635 Inst.setOpcode(NewOpcode); 1636 1637 // Replace the first operand and preserve auxiliary operands of 1638 // the instruction. 1639 Inst.erase(Inst.begin()); 1640 Inst.insert(Inst.begin(), 1641 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 1642 Inst.insert(Inst.begin(), 1643 MCOperand::createExpr( // Displacement 1644 MCSymbolRefExpr::create(TargetLocation, 1645 MCSymbolRefExpr::VK_None, *Ctx))); 1646 Inst.insert(Inst.begin(), 1647 MCOperand::createReg(X86::NoRegister)); // IndexReg 1648 Inst.insert(Inst.begin(), 1649 MCOperand::createImm(1)); // ScaleAmt 1650 Inst.insert(Inst.begin(), 1651 MCOperand::createReg(X86::RIP)); // BaseReg 1652 1653 return true; 1654 } 1655 1656 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { 1657 bool IsTailCall = isTailCall(Inst); 1658 if (IsTailCall) 1659 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1660 if (Inst.getOpcode() == X86::CALL64m || 1661 (Inst.getOpcode() == X86::JMP32m && IsTailCall)) { 1662 Inst.setOpcode(X86::MOV64rm); 1663 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1664 return; 1665 } 1666 if (Inst.getOpcode() == X86::CALL64r || 1667 (Inst.getOpcode() == X86::JMP32r && IsTailCall)) { 1668 Inst.setOpcode(X86::MOV64rr); 1669 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1670 return; 1671 } 1672 LLVM_DEBUG(Inst.dump()); 1673 llvm_unreachable("not implemented"); 1674 } 1675 1676 bool shortenInstruction(MCInst &Inst, 1677 const MCSubtargetInfo &STI) const override { 1678 unsigned OldOpcode = Inst.getOpcode(); 1679 unsigned NewOpcode = OldOpcode; 1680 1681 int MemOpNo = getMemoryOperandNo(Inst); 1682 1683 // Check and remove redundant Address-Size override prefix. 1684 if (opts::X86StripRedundantAddressSize) { 1685 uint64_t TSFlags = Info->get(OldOpcode).TSFlags; 1686 unsigned Flags = Inst.getFlags(); 1687 1688 if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) && 1689 Flags & X86::IP_HAS_AD_SIZE) 1690 Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE); 1691 } 1692 1693 // Check and remove EIZ/RIZ. These cases represent ambiguous cases where 1694 // SIB byte is present, but no index is used and modrm alone should have 1695 // been enough. Converting to NoRegister effectively removes the SIB byte. 1696 if (MemOpNo >= 0) { 1697 MCOperand &IndexOp = 1698 Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg); 1699 if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ) 1700 IndexOp = MCOperand::createReg(X86::NoRegister); 1701 } 1702 1703 if (isBranch(Inst)) { 1704 NewOpcode = getShortBranchOpcode(OldOpcode); 1705 } else if (OldOpcode == X86::MOV64ri) { 1706 if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1707 const int64_t Imm = 1708 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1709 if (int64_t(Imm) == int64_t(int32_t(Imm))) 1710 NewOpcode = X86::MOV64ri32; 1711 } 1712 } else { 1713 // If it's arithmetic instruction check if signed operand fits in 1 byte. 1714 const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(OldOpcode); 1715 if (ShortOpcode != OldOpcode && 1716 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1717 int64_t Imm = 1718 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1719 if (int64_t(Imm) == int64_t(int8_t(Imm))) 1720 NewOpcode = ShortOpcode; 1721 } 1722 } 1723 1724 if (NewOpcode == OldOpcode) 1725 return false; 1726 1727 Inst.setOpcode(NewOpcode); 1728 return true; 1729 } 1730 1731 bool 1732 convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp, 1733 bool AllowBasePtrStackMemOp) const override { 1734 // - Register-register moves are OK 1735 // - Stores are filtered out by opcode (no store CMOV) 1736 // - Non-stack loads are prohibited (generally unsafe) 1737 // - Stack loads are OK if AllowStackMemOp is true 1738 // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true 1739 if (isLoad(Inst)) { 1740 // If stack memory operands are not allowed, no loads are allowed 1741 if (!AllowStackMemOp) 1742 return false; 1743 1744 // If stack memory operands are allowed, check if it's a load from stack 1745 bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed; 1746 MCPhysReg Reg; 1747 int32_t SrcImm; 1748 uint16_t StackPtrReg; 1749 int64_t StackOffset; 1750 uint8_t Size; 1751 bool IsStackAccess = 1752 isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, 1753 StackPtrReg, StackOffset, Size, IsSimple, IsIndexed); 1754 // Prohibit non-stack-based loads 1755 if (!IsStackAccess) 1756 return false; 1757 // If stack memory operands are allowed, check if it's RBP-based 1758 if (!AllowBasePtrStackMemOp && 1759 RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg)) 1760 return false; 1761 } 1762 1763 unsigned NewOpcode = 0; 1764 switch (Inst.getOpcode()) { 1765 case X86::MOV16rr: 1766 NewOpcode = X86::CMOV16rr; 1767 break; 1768 case X86::MOV16rm: 1769 NewOpcode = X86::CMOV16rm; 1770 break; 1771 case X86::MOV32rr: 1772 NewOpcode = X86::CMOV32rr; 1773 break; 1774 case X86::MOV32rm: 1775 NewOpcode = X86::CMOV32rm; 1776 break; 1777 case X86::MOV64rr: 1778 NewOpcode = X86::CMOV64rr; 1779 break; 1780 case X86::MOV64rm: 1781 NewOpcode = X86::CMOV64rm; 1782 break; 1783 default: 1784 return false; 1785 } 1786 Inst.setOpcode(NewOpcode); 1787 // Insert CC at the end of prime operands, before annotations 1788 Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst), 1789 MCOperand::createImm(CC)); 1790 // CMOV is a 3-operand MCInst, so duplicate the destination as src1 1791 Inst.insert(Inst.begin(), Inst.getOperand(0)); 1792 return true; 1793 } 1794 1795 bool lowerTailCall(MCInst &Inst) override { 1796 if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) { 1797 Inst.setOpcode(X86::JMP_1); 1798 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1799 return true; 1800 } 1801 return false; 1802 } 1803 1804 const MCSymbol *getTargetSymbol(const MCInst &Inst, 1805 unsigned OpNum = 0) const override { 1806 if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) 1807 return nullptr; 1808 1809 const MCOperand &Op = Inst.getOperand(OpNum); 1810 if (!Op.isExpr()) 1811 return nullptr; 1812 1813 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 1814 if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) 1815 return nullptr; 1816 1817 return &SymExpr->getSymbol(); 1818 } 1819 1820 // This is the same as the base class, but since we are overriding one of 1821 // getTargetSymbol's signatures above, we need to override all of them. 1822 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { 1823 return &cast<const MCSymbolRefExpr>(Expr)->getSymbol(); 1824 } 1825 1826 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, 1827 const MCSymbol *&TBB, const MCSymbol *&FBB, 1828 MCInst *&CondBranch, 1829 MCInst *&UncondBranch) const override { 1830 auto I = End; 1831 1832 // Bottom-up analysis 1833 while (I != Begin) { 1834 --I; 1835 1836 // Ignore nops and CFIs 1837 if (isPseudo(*I)) 1838 continue; 1839 1840 // Stop when we find the first non-terminator 1841 if (!isTerminator(*I)) 1842 break; 1843 1844 if (!isBranch(*I)) 1845 break; 1846 1847 // Handle unconditional branches. 1848 if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 || 1849 I->getOpcode() == X86::JMP_4) && 1850 !isTailCall(*I)) { 1851 // If any code was seen after this unconditional branch, we've seen 1852 // unreachable code. Ignore them. 1853 CondBranch = nullptr; 1854 UncondBranch = &*I; 1855 const MCSymbol *Sym = getTargetSymbol(*I); 1856 assert(Sym != nullptr && 1857 "Couldn't extract BB symbol from jump operand"); 1858 TBB = Sym; 1859 continue; 1860 } 1861 1862 // Handle conditional branches and ignore indirect branches 1863 if (!isUnsupportedBranch(*I) && getCondCode(*I) == X86::COND_INVALID) { 1864 // Indirect branch 1865 return false; 1866 } 1867 1868 if (CondBranch == nullptr) { 1869 const MCSymbol *TargetBB = getTargetSymbol(*I); 1870 if (TargetBB == nullptr) { 1871 // Unrecognized branch target 1872 return false; 1873 } 1874 FBB = TBB; 1875 TBB = TargetBB; 1876 CondBranch = &*I; 1877 continue; 1878 } 1879 1880 llvm_unreachable("multiple conditional branches in one BB"); 1881 } 1882 return true; 1883 } 1884 1885 template <typename Itr> 1886 std::pair<IndirectBranchType, MCInst *> 1887 analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const { 1888 // Analyze PIC-style jump table code template: 1889 // 1890 // lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr 1891 // mov ({%r1|%r2}, %index, 4), {%r2|%r1} 1892 // add %r2, %r1 1893 // jmp *%r1 1894 // 1895 // (with any irrelevant instructions in-between) 1896 // 1897 // When we call this helper we've already determined %r1 and %r2, and 1898 // reverse instruction iterator \p II is pointing to the ADD instruction. 1899 // 1900 // PIC jump table looks like following: 1901 // 1902 // JT: ---------- 1903 // E1:| L1 - JT | 1904 // |----------| 1905 // E2:| L2 - JT | 1906 // |----------| 1907 // | | 1908 // ...... 1909 // En:| Ln - JT | 1910 // ---------- 1911 // 1912 // Where L1, L2, ..., Ln represent labels in the function. 1913 // 1914 // The actual relocations in the table will be of the form: 1915 // 1916 // Ln - JT 1917 // = (Ln - En) + (En - JT) 1918 // = R_X86_64_PC32(Ln) + En - JT 1919 // = R_X86_64_PC32(Ln + offsetof(En)) 1920 // 1921 LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n"); 1922 MCInst *MemLocInstr = nullptr; 1923 const MCInst *MovInstr = nullptr; 1924 while (++II != IE) { 1925 MCInst &Instr = *II; 1926 const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode()); 1927 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) && 1928 !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) { 1929 // Ignore instructions that don't affect R1, R2 registers. 1930 continue; 1931 } 1932 if (!MovInstr) { 1933 // Expect to see MOV instruction. 1934 if (!isMOVSX64rm32(Instr)) { 1935 LLVM_DEBUG(dbgs() << "MOV instruction expected.\n"); 1936 break; 1937 } 1938 1939 // Check if it's setting %r1 or %r2. In canonical form it sets %r2. 1940 // If it sets %r1 - rename the registers so we have to only check 1941 // a single form. 1942 unsigned MovDestReg = Instr.getOperand(0).getReg(); 1943 if (MovDestReg != R2) 1944 std::swap(R1, R2); 1945 if (MovDestReg != R2) { 1946 LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n"); 1947 break; 1948 } 1949 1950 // Verify operands for MOV. 1951 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1952 if (!MO) 1953 break; 1954 if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 || 1955 MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 || 1956 MO->SegRegNum != X86::NoRegister) 1957 break; 1958 MovInstr = &Instr; 1959 } else { 1960 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo)) 1961 continue; 1962 if (!isLEA64r(Instr)) { 1963 LLVM_DEBUG(dbgs() << "LEA instruction expected\n"); 1964 break; 1965 } 1966 if (Instr.getOperand(0).getReg() != R1) { 1967 LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n"); 1968 break; 1969 } 1970 1971 // Verify operands for LEA. 1972 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1973 if (!MO) 1974 break; 1975 if (MO->BaseRegNum != RegInfo->getProgramCounter() || 1976 MO->IndexRegNum != X86::NoRegister || 1977 MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr) 1978 break; 1979 MemLocInstr = &Instr; 1980 break; 1981 } 1982 } 1983 1984 if (!MemLocInstr) 1985 return std::make_pair(IndirectBranchType::UNKNOWN, nullptr); 1986 1987 LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n"); 1988 return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE, 1989 MemLocInstr); 1990 } 1991 1992 IndirectBranchType analyzeIndirectBranch( 1993 MCInst &Instruction, InstructionIterator Begin, InstructionIterator End, 1994 const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut, 1995 unsigned &IndexRegNumOut, int64_t &DispValueOut, 1996 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override { 1997 // Try to find a (base) memory location from where the address for 1998 // the indirect branch is loaded. For X86-64 the memory will be specified 1999 // in the following format: 2000 // 2001 // {%rip}/{%basereg} + Imm + IndexReg * Scale 2002 // 2003 // We are interested in the cases where Scale == sizeof(uintptr_t) and 2004 // the contents of the memory are presumably an array of pointers to code. 2005 // 2006 // Normal jump table: 2007 // 2008 // jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr 2009 // 2010 // or 2011 // 2012 // mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr 2013 // ... 2014 // jmp %r1 2015 // 2016 // We handle PIC-style jump tables separately. 2017 // 2018 MemLocInstrOut = nullptr; 2019 BaseRegNumOut = X86::NoRegister; 2020 IndexRegNumOut = X86::NoRegister; 2021 DispValueOut = 0; 2022 DispExprOut = nullptr; 2023 2024 std::reverse_iterator<InstructionIterator> II(End); 2025 std::reverse_iterator<InstructionIterator> IE(Begin); 2026 2027 IndirectBranchType Type = IndirectBranchType::UNKNOWN; 2028 2029 // An instruction referencing memory used by jump instruction (directly or 2030 // via register). This location could be an array of function pointers 2031 // in case of indirect tail call, or a jump table. 2032 MCInst *MemLocInstr = nullptr; 2033 2034 if (MCPlus::getNumPrimeOperands(Instruction) == 1) { 2035 // If the indirect jump is on register - try to detect if the 2036 // register value is loaded from a memory location. 2037 assert(Instruction.getOperand(0).isReg() && "register operand expected"); 2038 const unsigned R1 = Instruction.getOperand(0).getReg(); 2039 // Check if one of the previous instructions defines the jump-on register. 2040 for (auto PrevII = II; PrevII != IE; ++PrevII) { 2041 MCInst &PrevInstr = *PrevII; 2042 const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode()); 2043 2044 if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo)) 2045 continue; 2046 2047 if (isMoveMem2Reg(PrevInstr)) { 2048 MemLocInstr = &PrevInstr; 2049 break; 2050 } 2051 if (isADD64rr(PrevInstr)) { 2052 unsigned R2 = PrevInstr.getOperand(2).getReg(); 2053 if (R1 == R2) 2054 return IndirectBranchType::UNKNOWN; 2055 std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2); 2056 break; 2057 } 2058 return IndirectBranchType::UNKNOWN; 2059 } 2060 if (!MemLocInstr) { 2061 // No definition seen for the register in this function so far. Could be 2062 // an input parameter - which means it is an external code reference. 2063 // It also could be that the definition happens to be in the code that 2064 // we haven't processed yet. Since we have to be conservative, return 2065 // as UNKNOWN case. 2066 return IndirectBranchType::UNKNOWN; 2067 } 2068 } else { 2069 MemLocInstr = &Instruction; 2070 } 2071 2072 const MCRegister RIPRegister = RegInfo->getProgramCounter(); 2073 2074 // Analyze the memory location. 2075 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr); 2076 if (!MO) 2077 return IndirectBranchType::UNKNOWN; 2078 2079 BaseRegNumOut = MO->BaseRegNum; 2080 IndexRegNumOut = MO->IndexRegNum; 2081 DispValueOut = MO->DispImm; 2082 DispExprOut = MO->DispExpr; 2083 2084 if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) || 2085 MO->SegRegNum != X86::NoRegister) 2086 return IndirectBranchType::UNKNOWN; 2087 2088 if (MemLocInstr == &Instruction && 2089 (!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) { 2090 MemLocInstrOut = MemLocInstr; 2091 return IndirectBranchType::POSSIBLE_FIXED_BRANCH; 2092 } 2093 2094 if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2095 (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)) 2096 return IndirectBranchType::UNKNOWN; 2097 2098 if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2099 MO->ScaleImm != PtrSize) 2100 return IndirectBranchType::UNKNOWN; 2101 2102 MemLocInstrOut = MemLocInstr; 2103 2104 return Type; 2105 } 2106 2107 /// Analyze a callsite to see if it could be a virtual method call. This only 2108 /// checks to see if the overall pattern is satisfied, it does not guarantee 2109 /// that the callsite is a true virtual method call. 2110 /// The format of virtual method calls that are recognized is one of the 2111 /// following: 2112 /// 2113 /// Form 1: (found in debug code) 2114 /// add METHOD_OFFSET, %VtableReg 2115 /// mov (%VtableReg), %MethodReg 2116 /// ... 2117 /// call or jmp *%MethodReg 2118 /// 2119 /// Form 2: 2120 /// mov METHOD_OFFSET(%VtableReg), %MethodReg 2121 /// ... 2122 /// call or jmp *%MethodReg 2123 /// 2124 /// Form 3: 2125 /// ... 2126 /// call or jmp *METHOD_OFFSET(%VtableReg) 2127 /// 2128 bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin, 2129 InstructionIterator ForwardEnd, 2130 std::vector<MCInst *> &MethodFetchInsns, 2131 unsigned &VtableRegNum, unsigned &MethodRegNum, 2132 uint64_t &MethodOffset) const override { 2133 VtableRegNum = X86::NoRegister; 2134 MethodRegNum = X86::NoRegister; 2135 MethodOffset = 0; 2136 2137 std::reverse_iterator<InstructionIterator> Itr(ForwardEnd); 2138 std::reverse_iterator<InstructionIterator> End(ForwardBegin); 2139 2140 MCInst &CallInst = *Itr++; 2141 assert(isIndirectBranch(CallInst) || isCall(CallInst)); 2142 2143 // The call can just be jmp offset(reg) 2144 if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) { 2145 if (!MO->DispExpr && MO->BaseRegNum != X86::RIP && 2146 MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) { 2147 MethodRegNum = MO->BaseRegNum; 2148 if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister && 2149 MO->SegRegNum == X86::NoRegister) { 2150 VtableRegNum = MethodRegNum; 2151 MethodOffset = MO->DispImm; 2152 MethodFetchInsns.push_back(&CallInst); 2153 return true; 2154 } 2155 } 2156 return false; 2157 } 2158 if (CallInst.getOperand(0).isReg()) 2159 MethodRegNum = CallInst.getOperand(0).getReg(); 2160 else 2161 return false; 2162 2163 if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) { 2164 VtableRegNum = X86::NoRegister; 2165 MethodRegNum = X86::NoRegister; 2166 return false; 2167 } 2168 2169 // find load from vtable, this may or may not include the method offset 2170 while (Itr != End) { 2171 MCInst &CurInst = *Itr++; 2172 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2173 if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) { 2174 if (!isLoad(CurInst)) 2175 return false; 2176 if (std::optional<X86MemOperand> MO = 2177 evaluateX86MemoryOperand(CurInst)) { 2178 if (!MO->DispExpr && MO->ScaleImm == 1 && 2179 MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP && 2180 MO->BaseRegNum != X86::NoRegister && 2181 MO->IndexRegNum == X86::NoRegister && 2182 MO->SegRegNum == X86::NoRegister && MO->BaseRegNum != X86::RIP) { 2183 VtableRegNum = MO->BaseRegNum; 2184 MethodOffset = MO->DispImm; 2185 MethodFetchInsns.push_back(&CurInst); 2186 if (MethodOffset != 0) 2187 return true; 2188 break; 2189 } 2190 } 2191 return false; 2192 } 2193 } 2194 2195 if (!VtableRegNum) 2196 return false; 2197 2198 // look for any adds affecting the method register. 2199 while (Itr != End) { 2200 MCInst &CurInst = *Itr++; 2201 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2202 if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) { 2203 if (isADDri(CurInst)) { 2204 assert(!MethodOffset); 2205 MethodOffset = CurInst.getOperand(2).getImm(); 2206 MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst); 2207 break; 2208 } 2209 } 2210 } 2211 2212 return true; 2213 } 2214 2215 bool createStackPointerIncrement(MCInst &Inst, int Size, 2216 bool NoFlagsClobber) const override { 2217 if (NoFlagsClobber) { 2218 Inst.setOpcode(X86::LEA64r); 2219 Inst.clear(); 2220 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2221 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2222 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2223 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2224 Inst.addOperand(MCOperand::createImm(-Size)); // Displacement 2225 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2226 return true; 2227 } 2228 Inst.setOpcode(X86::SUB64ri8); 2229 Inst.clear(); 2230 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2231 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2232 Inst.addOperand(MCOperand::createImm(Size)); 2233 return true; 2234 } 2235 2236 bool createStackPointerDecrement(MCInst &Inst, int Size, 2237 bool NoFlagsClobber) const override { 2238 if (NoFlagsClobber) { 2239 Inst.setOpcode(X86::LEA64r); 2240 Inst.clear(); 2241 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2242 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2243 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2244 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2245 Inst.addOperand(MCOperand::createImm(Size)); // Displacement 2246 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2247 return true; 2248 } 2249 Inst.setOpcode(X86::ADD64ri8); 2250 Inst.clear(); 2251 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2252 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2253 Inst.addOperand(MCOperand::createImm(Size)); 2254 return true; 2255 } 2256 2257 bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset, 2258 const MCPhysReg &SrcReg, int Size) const override { 2259 unsigned NewOpcode; 2260 switch (Size) { 2261 default: 2262 return false; 2263 case 2: NewOpcode = X86::MOV16mr; break; 2264 case 4: NewOpcode = X86::MOV32mr; break; 2265 case 8: NewOpcode = X86::MOV64mr; break; 2266 } 2267 Inst.setOpcode(NewOpcode); 2268 Inst.clear(); 2269 Inst.addOperand(MCOperand::createReg(StackReg)); // BaseReg 2270 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2271 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2272 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2273 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2274 Inst.addOperand(MCOperand::createReg(SrcReg)); 2275 return true; 2276 } 2277 2278 bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, 2279 int Offset, const MCPhysReg &DstReg, 2280 int Size) const override { 2281 return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister, 2282 Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister, 2283 DstReg, Size); 2284 } 2285 2286 bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale, 2287 const MCPhysReg &IndexReg, int64_t Offset, 2288 const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg, 2289 const MCPhysReg &DstReg, int Size) const override { 2290 unsigned NewOpcode; 2291 switch (Size) { 2292 default: 2293 return false; 2294 case 2: NewOpcode = X86::MOV16rm; break; 2295 case 4: NewOpcode = X86::MOV32rm; break; 2296 case 8: NewOpcode = X86::MOV64rm; break; 2297 } 2298 Inst.setOpcode(NewOpcode); 2299 Inst.clear(); 2300 Inst.addOperand(MCOperand::createReg(DstReg)); 2301 Inst.addOperand(MCOperand::createReg(BaseReg)); 2302 Inst.addOperand(MCOperand::createImm(Scale)); 2303 Inst.addOperand(MCOperand::createReg(IndexReg)); 2304 if (OffsetExpr) 2305 Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement 2306 else 2307 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2308 Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg 2309 return true; 2310 } 2311 2312 void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest, 2313 uint32_t Imm) const override { 2314 Inst.setOpcode(X86::MOV64ri32); 2315 Inst.clear(); 2316 Inst.addOperand(MCOperand::createReg(Dest)); 2317 Inst.addOperand(MCOperand::createImm(Imm)); 2318 } 2319 2320 bool createIncMemory(MCInst &Inst, const MCSymbol *Target, 2321 MCContext *Ctx) const override { 2322 2323 Inst.setOpcode(X86::LOCK_INC64m); 2324 Inst.clear(); 2325 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 2326 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2327 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2328 2329 Inst.addOperand(MCOperand::createExpr( 2330 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, 2331 *Ctx))); // Displacement 2332 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2333 return true; 2334 } 2335 2336 bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts, 2337 const MCOperand &BaseReg, const MCOperand &Scale, 2338 const MCOperand &IndexReg, const MCOperand &Offset, 2339 const MCOperand &TmpReg) const override { 2340 // The code fragment we emit here is: 2341 // 2342 // mov32 (%base, %index, scale), %tmpreg 2343 // ijmp *(%tmpreg) 2344 // 2345 MCInst IJmp; 2346 IJmp.setOpcode(X86::JMP64r); 2347 IJmp.addOperand(TmpReg); 2348 2349 MCInst Load; 2350 Load.setOpcode(X86::MOV32rm); 2351 Load.addOperand(TmpReg); 2352 Load.addOperand(BaseReg); 2353 Load.addOperand(Scale); 2354 Load.addOperand(IndexReg); 2355 Load.addOperand(Offset); 2356 Load.addOperand(MCOperand::createReg(X86::NoRegister)); 2357 2358 Insts.push_back(Load); 2359 Insts.push_back(IJmp); 2360 return true; 2361 } 2362 2363 bool createNoop(MCInst &Inst) const override { 2364 Inst.setOpcode(X86::NOOP); 2365 return true; 2366 } 2367 2368 bool createReturn(MCInst &Inst) const override { 2369 Inst.setOpcode(X86::RET64); 2370 return true; 2371 } 2372 2373 InstructionListType createInlineMemcpy(bool ReturnEnd) const override { 2374 InstructionListType Code; 2375 if (ReturnEnd) 2376 Code.emplace_back(MCInstBuilder(X86::LEA64r) 2377 .addReg(X86::RAX) 2378 .addReg(X86::RDI) 2379 .addImm(1) 2380 .addReg(X86::RDX) 2381 .addImm(0) 2382 .addReg(X86::NoRegister)); 2383 else 2384 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2385 .addReg(X86::RAX) 2386 .addReg(X86::RDI)); 2387 2388 Code.emplace_back(MCInstBuilder(X86::MOV32rr) 2389 .addReg(X86::ECX) 2390 .addReg(X86::EDX)); 2391 Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64)); 2392 2393 return Code; 2394 } 2395 2396 InstructionListType createOneByteMemcpy() const override { 2397 InstructionListType Code; 2398 Code.emplace_back(MCInstBuilder(X86::MOV8rm) 2399 .addReg(X86::CL) 2400 .addReg(X86::RSI) 2401 .addImm(0) 2402 .addReg(X86::NoRegister) 2403 .addImm(0) 2404 .addReg(X86::NoRegister)); 2405 Code.emplace_back(MCInstBuilder(X86::MOV8mr) 2406 .addReg(X86::RDI) 2407 .addImm(0) 2408 .addReg(X86::NoRegister) 2409 .addImm(0) 2410 .addReg(X86::NoRegister) 2411 .addReg(X86::CL)); 2412 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2413 .addReg(X86::RAX) 2414 .addReg(X86::RDI)); 2415 return Code; 2416 } 2417 2418 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, 2419 const MCSymbol *Target, 2420 MCContext *Ctx) const override { 2421 InstructionListType Code; 2422 Code.emplace_back(MCInstBuilder(X86::CMP64ri8) 2423 .addReg(RegNo) 2424 .addImm(Imm)); 2425 Code.emplace_back(MCInstBuilder(X86::JCC_1) 2426 .addExpr(MCSymbolRefExpr::create( 2427 Target, MCSymbolRefExpr::VK_None, *Ctx)) 2428 .addImm(X86::COND_E)); 2429 return Code; 2430 } 2431 2432 std::optional<Relocation> 2433 createRelocation(const MCFixup &Fixup, 2434 const MCAsmBackend &MAB) const override { 2435 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); 2436 2437 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); 2438 const uint64_t RelOffset = Fixup.getOffset(); 2439 2440 uint64_t RelType; 2441 if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) { 2442 switch (FKI.TargetSize) { 2443 default: 2444 return std::nullopt; 2445 case 8: RelType = ELF::R_X86_64_PC8; break; 2446 case 16: RelType = ELF::R_X86_64_PC16; break; 2447 case 32: RelType = ELF::R_X86_64_PC32; break; 2448 case 64: RelType = ELF::R_X86_64_PC64; break; 2449 } 2450 } else { 2451 switch (FKI.TargetSize) { 2452 default: 2453 return std::nullopt; 2454 case 8: RelType = ELF::R_X86_64_8; break; 2455 case 16: RelType = ELF::R_X86_64_16; break; 2456 case 32: RelType = ELF::R_X86_64_32; break; 2457 case 64: RelType = ELF::R_X86_64_64; break; 2458 } 2459 } 2460 2461 // Extract a symbol and an addend out of the fixup value expression. 2462 // 2463 // Only the following limited expression types are supported: 2464 // Symbol + Addend 2465 // Symbol + Constant + Addend 2466 // Const + Addend 2467 // Symbol 2468 uint64_t Addend = 0; 2469 MCSymbol *Symbol = nullptr; 2470 const MCExpr *ValueExpr = Fixup.getValue(); 2471 if (ValueExpr->getKind() == MCExpr::Binary) { 2472 const auto *BinaryExpr = cast<MCBinaryExpr>(ValueExpr); 2473 assert(BinaryExpr->getOpcode() == MCBinaryExpr::Add && 2474 "unexpected binary expression"); 2475 const MCExpr *LHS = BinaryExpr->getLHS(); 2476 if (LHS->getKind() == MCExpr::Constant) { 2477 Addend = cast<MCConstantExpr>(LHS)->getValue(); 2478 } else if (LHS->getKind() == MCExpr::Binary) { 2479 const auto *LHSBinaryExpr = cast<MCBinaryExpr>(LHS); 2480 assert(LHSBinaryExpr->getOpcode() == MCBinaryExpr::Add && 2481 "unexpected binary expression"); 2482 const MCExpr *LLHS = LHSBinaryExpr->getLHS(); 2483 assert(LLHS->getKind() == MCExpr::SymbolRef && "unexpected LLHS"); 2484 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LLHS)); 2485 const MCExpr *RLHS = LHSBinaryExpr->getRHS(); 2486 assert(RLHS->getKind() == MCExpr::Constant && "unexpected RLHS"); 2487 Addend = cast<MCConstantExpr>(RLHS)->getValue(); 2488 } else { 2489 assert(LHS->getKind() == MCExpr::SymbolRef && "unexpected LHS"); 2490 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(LHS)); 2491 } 2492 const MCExpr *RHS = BinaryExpr->getRHS(); 2493 assert(RHS->getKind() == MCExpr::Constant && "unexpected RHS"); 2494 Addend += cast<MCConstantExpr>(RHS)->getValue(); 2495 } else { 2496 assert(ValueExpr->getKind() == MCExpr::SymbolRef && "unexpected value"); 2497 Symbol = const_cast<MCSymbol *>(this->getTargetSymbol(ValueExpr)); 2498 } 2499 2500 return Relocation({RelOffset, Symbol, RelType, Addend, 0}); 2501 } 2502 2503 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, 2504 int64_t Addend, MCContext *Ctx, int64_t &Value, 2505 uint64_t RelType) const override { 2506 unsigned ImmOpNo = -1U; 2507 2508 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 2509 ++Index) { 2510 if (Inst.getOperand(Index).isImm()) { 2511 ImmOpNo = Index; 2512 // TODO: this is a bit hacky. It finds the correct operand by 2513 // searching for a specific immediate value. If no value is 2514 // provided it defaults to the last immediate operand found. 2515 // This could lead to unexpected results if the instruction 2516 // has more than one immediate with the same value. 2517 if (Inst.getOperand(ImmOpNo).getImm() == Value) 2518 break; 2519 } 2520 } 2521 2522 if (ImmOpNo == -1U) 2523 return false; 2524 2525 Value = Inst.getOperand(ImmOpNo).getImm(); 2526 2527 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType); 2528 2529 return true; 2530 } 2531 2532 bool replaceRegWithImm(MCInst &Inst, unsigned Register, 2533 int64_t Imm) const override { 2534 2535 enum CheckSignExt : uint8_t { 2536 NOCHECK = 0, 2537 CHECK8, 2538 CHECK32, 2539 }; 2540 2541 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 2542 struct InstInfo { 2543 // Size in bytes that Inst loads from memory. 2544 uint8_t DataSize; 2545 2546 // True when the target operand has to be duplicated because the opcode 2547 // expects a LHS operand. 2548 bool HasLHS; 2549 2550 // List of checks and corresponding opcodes to be used. We try to use the 2551 // smallest possible immediate value when various sizes are available, 2552 // hence we may need to check whether a larger constant fits in a smaller 2553 // immediate. 2554 CheckList Checks; 2555 }; 2556 2557 InstInfo I; 2558 2559 switch (Inst.getOpcode()) { 2560 default: { 2561 switch (getPushSize(Inst)) { 2562 2563 case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break; 2564 case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break; 2565 case 8: I = {8, false, {{CHECK8, X86::PUSH64i8}, 2566 {CHECK32, X86::PUSH64i32}, 2567 {NOCHECK, Inst.getOpcode()}}}; break; 2568 default: return false; 2569 } 2570 break; 2571 } 2572 2573 // MOV 2574 case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 2575 case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 2576 case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 2577 case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32}, 2578 {NOCHECK, X86::MOV64ri}}}; break; 2579 2580 case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break; 2581 case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break; 2582 case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break; 2583 case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32}, 2584 {NOCHECK, X86::MOV64mr}}}; break; 2585 2586 // MOVZX 2587 case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 2588 case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 2589 case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 2590 2591 // CMP 2592 case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 2593 case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8}, 2594 {NOCHECK, X86::CMP16ri}}}; break; 2595 case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8}, 2596 {NOCHECK, X86::CMP32ri}}}; break; 2597 case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8}, 2598 {CHECK32, X86::CMP64ri32}, 2599 {NOCHECK, X86::CMP64rr}}}; break; 2600 2601 // TEST 2602 case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 2603 case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 2604 case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 2605 case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 2606 {NOCHECK, X86::TEST64rr}}}; break; 2607 2608 // ADD 2609 case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 2610 case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8}, 2611 {NOCHECK, X86::ADD16ri}}}; break; 2612 case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8}, 2613 {NOCHECK, X86::ADD32ri}}}; break; 2614 case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8}, 2615 {CHECK32, X86::ADD64ri32}, 2616 {NOCHECK, X86::ADD64rr}}}; break; 2617 2618 // SUB 2619 case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 2620 case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8}, 2621 {NOCHECK, X86::SUB16ri}}}; break; 2622 case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8}, 2623 {NOCHECK, X86::SUB32ri}}}; break; 2624 case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8}, 2625 {CHECK32, X86::SUB64ri32}, 2626 {NOCHECK, X86::SUB64rr}}}; break; 2627 2628 // AND 2629 case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 2630 case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8}, 2631 {NOCHECK, X86::AND16ri}}}; break; 2632 case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8}, 2633 {NOCHECK, X86::AND32ri}}}; break; 2634 case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8}, 2635 {CHECK32, X86::AND64ri32}, 2636 {NOCHECK, X86::AND64rr}}}; break; 2637 2638 // OR 2639 case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 2640 case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8}, 2641 {NOCHECK, X86::OR16ri}}}; break; 2642 case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8}, 2643 {NOCHECK, X86::OR32ri}}}; break; 2644 case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8}, 2645 {CHECK32, X86::OR64ri32}, 2646 {NOCHECK, X86::OR64rr}}}; break; 2647 2648 // XOR 2649 case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 2650 case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8}, 2651 {NOCHECK, X86::XOR16ri}}}; break; 2652 case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8}, 2653 {NOCHECK, X86::XOR32ri}}}; break; 2654 case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8}, 2655 {CHECK32, X86::XOR64ri32}, 2656 {NOCHECK, X86::XOR64rr}}}; break; 2657 } 2658 2659 // Compute the new opcode. 2660 unsigned NewOpcode = 0; 2661 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 2662 NewOpcode = Check.second; 2663 if (Check.first == NOCHECK) 2664 break; 2665 if (Check.first == CHECK8 && isInt<8>(Imm)) 2666 break; 2667 if (Check.first == CHECK32 && isInt<32>(Imm)) 2668 break; 2669 } 2670 if (NewOpcode == Inst.getOpcode()) 2671 return false; 2672 2673 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2674 2675 unsigned NumFound = 0; 2676 for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0), 2677 E = InstDesc.getNumOperands(); 2678 Index != E; ++Index) 2679 if (Inst.getOperand(Index).isReg() && 2680 Inst.getOperand(Index).getReg() == Register) 2681 NumFound++; 2682 2683 if (NumFound != 1) 2684 return false; 2685 2686 MCOperand TargetOp = Inst.getOperand(0); 2687 Inst.clear(); 2688 Inst.setOpcode(NewOpcode); 2689 Inst.addOperand(TargetOp); 2690 if (I.HasLHS) 2691 Inst.addOperand(TargetOp); 2692 Inst.addOperand(MCOperand::createImm(Imm)); 2693 2694 return true; 2695 } 2696 2697 bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace, 2698 unsigned ReplaceWith) const override { 2699 2700 // Get the HasLHS value so that iteration can be done 2701 bool HasLHS; 2702 if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) || 2703 X86::isSUB(Inst.getOpcode())) { 2704 HasLHS = true; 2705 } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) || 2706 X86::isTEST(Inst.getOpcode())) { 2707 HasLHS = false; 2708 } else { 2709 switch (Inst.getOpcode()) { 2710 case X86::MOV8rr: 2711 case X86::MOV8rm: 2712 case X86::MOV8mr: 2713 case X86::MOV8ri: 2714 case X86::MOV16rr: 2715 case X86::MOV16rm: 2716 case X86::MOV16mr: 2717 case X86::MOV16ri: 2718 case X86::MOV32rr: 2719 case X86::MOV32rm: 2720 case X86::MOV32mr: 2721 case X86::MOV32ri: 2722 case X86::MOV64rr: 2723 case X86::MOV64rm: 2724 case X86::MOV64mr: 2725 case X86::MOV64ri: 2726 case X86::MOVZX16rr8: 2727 case X86::MOVZX32rr8: 2728 case X86::MOVZX32rr16: 2729 case X86::MOVSX32rm8: 2730 case X86::MOVSX32rr8: 2731 case X86::MOVSX64rm32: 2732 case X86::LEA64r: 2733 HasLHS = false; 2734 break; 2735 default: 2736 return false; 2737 } 2738 } 2739 2740 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2741 2742 bool FoundOne = false; 2743 2744 // Iterate only through src operands that arent also dest operands 2745 for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0), 2746 E = InstDesc.getNumOperands(); 2747 Index != E; ++Index) { 2748 BitVector RegAliases = getAliases(ToReplace, true); 2749 if (!Inst.getOperand(Index).isReg() || 2750 !RegAliases.test(Inst.getOperand(Index).getReg())) 2751 continue; 2752 // Resize register if needed 2753 unsigned SizedReplaceWith = getAliasSized( 2754 ReplaceWith, getRegSize(Inst.getOperand(Index).getReg())); 2755 MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith); 2756 Inst.getOperand(Index) = NewOperand; 2757 FoundOne = true; 2758 } 2759 2760 // Return true if at least one operand was replaced 2761 return FoundOne; 2762 } 2763 2764 bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, 2765 MCContext *Ctx) const override { 2766 Inst.setOpcode(X86::JMP_1); 2767 Inst.addOperand(MCOperand::createExpr( 2768 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx))); 2769 return true; 2770 } 2771 2772 bool createCall(MCInst &Inst, const MCSymbol *Target, 2773 MCContext *Ctx) override { 2774 Inst.setOpcode(X86::CALL64pcrel32); 2775 Inst.addOperand(MCOperand::createExpr( 2776 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2777 return true; 2778 } 2779 2780 bool createTailCall(MCInst &Inst, const MCSymbol *Target, 2781 MCContext *Ctx) override { 2782 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); 2783 } 2784 2785 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, 2786 MCContext *Ctx) override { 2787 Seq.clear(); 2788 Seq.emplace_back(); 2789 createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true); 2790 } 2791 2792 bool createTrap(MCInst &Inst) const override { 2793 Inst.clear(); 2794 Inst.setOpcode(X86::TRAP); 2795 return true; 2796 } 2797 2798 bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, 2799 MCContext *Ctx) const override { 2800 unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); 2801 assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); 2802 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); 2803 Inst.getOperand(0) = MCOperand::createExpr( 2804 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2805 return true; 2806 } 2807 2808 bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx, 2809 unsigned CC) const override { 2810 if (CC == X86::COND_INVALID) 2811 return false; 2812 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC); 2813 Inst.getOperand(0) = MCOperand::createExpr( 2814 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2815 return true; 2816 } 2817 2818 unsigned getCanonicalBranchCondCode(unsigned CC) const override { 2819 switch (CC) { 2820 default: return X86::COND_INVALID; 2821 2822 case X86::COND_E: return X86::COND_E; 2823 case X86::COND_NE: return X86::COND_E; 2824 2825 case X86::COND_L: return X86::COND_L; 2826 case X86::COND_GE: return X86::COND_L; 2827 2828 case X86::COND_LE: return X86::COND_G; 2829 case X86::COND_G: return X86::COND_G; 2830 2831 case X86::COND_B: return X86::COND_B; 2832 case X86::COND_AE: return X86::COND_B; 2833 2834 case X86::COND_BE: return X86::COND_A; 2835 case X86::COND_A: return X86::COND_A; 2836 2837 case X86::COND_S: return X86::COND_S; 2838 case X86::COND_NS: return X86::COND_S; 2839 2840 case X86::COND_P: return X86::COND_P; 2841 case X86::COND_NP: return X86::COND_P; 2842 2843 case X86::COND_O: return X86::COND_O; 2844 case X86::COND_NO: return X86::COND_O; 2845 } 2846 } 2847 2848 bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, 2849 MCContext *Ctx) const override { 2850 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && 2851 "Invalid instruction"); 2852 Inst.getOperand(0) = MCOperand::createExpr( 2853 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2854 return true; 2855 } 2856 2857 MCPhysReg getX86R11() const override { return X86::R11; } 2858 2859 unsigned getShortBranchOpcode(unsigned Opcode) const override { 2860 switch (Opcode) { 2861 default: 2862 return Opcode; 2863 case X86::JMP_2: 2864 return X86::JMP_1; 2865 case X86::JMP_4: 2866 return X86::JMP_1; 2867 case X86::JCC_2: 2868 return X86::JCC_1; 2869 case X86::JCC_4: 2870 return X86::JCC_1; 2871 } 2872 } 2873 2874 MCPhysReg getIntArgRegister(unsigned ArgNo) const override { 2875 // FIXME: this should depend on the calling convention. 2876 switch (ArgNo) { 2877 case 0: return X86::RDI; 2878 case 1: return X86::RSI; 2879 case 2: return X86::RDX; 2880 case 3: return X86::RCX; 2881 case 4: return X86::R8; 2882 case 5: return X86::R9; 2883 default: return getNoRegister(); 2884 } 2885 } 2886 2887 void createPause(MCInst &Inst) const override { 2888 Inst.clear(); 2889 Inst.setOpcode(X86::PAUSE); 2890 } 2891 2892 void createLfence(MCInst &Inst) const override { 2893 Inst.clear(); 2894 Inst.setOpcode(X86::LFENCE); 2895 } 2896 2897 bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, 2898 bool IsTailCall) override { 2899 Inst.clear(); 2900 Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 2901 Inst.addOperand(MCOperand::createExpr( 2902 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2903 if (IsTailCall) 2904 setTailCall(Inst); 2905 return true; 2906 } 2907 2908 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, 2909 MCContext *Ctx, bool IsTailCall) override { 2910 Seq.clear(); 2911 MCInst Inst; 2912 Inst.setOpcode(X86::JMP_1); 2913 Inst.addOperand(MCOperand::createExpr( 2914 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2915 if (IsTailCall) 2916 setTailCall(Inst); 2917 Seq.emplace_back(Inst); 2918 } 2919 2920 bool isConditionalMove(const MCInst &Inst) const override { 2921 unsigned OpCode = Inst.getOpcode(); 2922 return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr || 2923 OpCode == X86::CMOV64rr); 2924 } 2925 2926 bool isBranchOnMem(const MCInst &Inst) const override { 2927 unsigned OpCode = Inst.getOpcode(); 2928 if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) || 2929 OpCode == X86::JMP64m) 2930 return true; 2931 2932 return false; 2933 } 2934 2935 bool isBranchOnReg(const MCInst &Inst) const override { 2936 unsigned OpCode = Inst.getOpcode(); 2937 if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) || 2938 OpCode == X86::JMP64r) 2939 return true; 2940 2941 return false; 2942 } 2943 2944 void createPushRegister(MCInst &Inst, MCPhysReg Reg, 2945 unsigned Size) const override { 2946 Inst.clear(); 2947 unsigned NewOpcode = 0; 2948 if (Reg == X86::EFLAGS) { 2949 switch (Size) { 2950 case 2: NewOpcode = X86::PUSHF16; break; 2951 case 4: NewOpcode = X86::PUSHF32; break; 2952 case 8: NewOpcode = X86::PUSHF64; break; 2953 default: 2954 llvm_unreachable("Unexpected size"); 2955 } 2956 Inst.setOpcode(NewOpcode); 2957 return; 2958 } 2959 switch (Size) { 2960 case 2: NewOpcode = X86::PUSH16r; break; 2961 case 4: NewOpcode = X86::PUSH32r; break; 2962 case 8: NewOpcode = X86::PUSH64r; break; 2963 default: 2964 llvm_unreachable("Unexpected size"); 2965 } 2966 Inst.setOpcode(NewOpcode); 2967 Inst.addOperand(MCOperand::createReg(Reg)); 2968 } 2969 2970 void createPopRegister(MCInst &Inst, MCPhysReg Reg, 2971 unsigned Size) const override { 2972 Inst.clear(); 2973 unsigned NewOpcode = 0; 2974 if (Reg == X86::EFLAGS) { 2975 switch (Size) { 2976 case 2: NewOpcode = X86::POPF16; break; 2977 case 4: NewOpcode = X86::POPF32; break; 2978 case 8: NewOpcode = X86::POPF64; break; 2979 default: 2980 llvm_unreachable("Unexpected size"); 2981 } 2982 Inst.setOpcode(NewOpcode); 2983 return; 2984 } 2985 switch (Size) { 2986 case 2: NewOpcode = X86::POP16r; break; 2987 case 4: NewOpcode = X86::POP32r; break; 2988 case 8: NewOpcode = X86::POP64r; break; 2989 default: 2990 llvm_unreachable("Unexpected size"); 2991 } 2992 Inst.setOpcode(NewOpcode); 2993 Inst.addOperand(MCOperand::createReg(Reg)); 2994 } 2995 2996 void createPushFlags(MCInst &Inst, unsigned Size) const override { 2997 return createPushRegister(Inst, X86::EFLAGS, Size); 2998 } 2999 3000 void createPopFlags(MCInst &Inst, unsigned Size) const override { 3001 return createPopRegister(Inst, X86::EFLAGS, Size); 3002 } 3003 3004 void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value, 3005 unsigned Size) const { 3006 unsigned int Opcode; 3007 switch (Size) { 3008 case 1: Opcode = X86::ADD8ri; break; 3009 case 2: Opcode = X86::ADD16ri; break; 3010 case 4: Opcode = X86::ADD32ri; break; 3011 default: 3012 llvm_unreachable("Unexpected size"); 3013 } 3014 Inst.setOpcode(Opcode); 3015 Inst.clear(); 3016 Inst.addOperand(MCOperand::createReg(Reg)); 3017 Inst.addOperand(MCOperand::createReg(Reg)); 3018 Inst.addOperand(MCOperand::createImm(Value)); 3019 } 3020 3021 void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg, 3022 unsigned Size) const { 3023 unsigned int Opcode; 3024 switch (Size) { 3025 case 1: Opcode = X86::MOV8ri; break; 3026 case 2: Opcode = X86::MOV16ri; break; 3027 case 4: Opcode = X86::MOV32ri; break; 3028 // Writing to a 32-bit register always zeros the upper 32 bits of the 3029 // full-width register 3030 case 8: 3031 Opcode = X86::MOV32ri; 3032 Reg = getAliasSized(Reg, 4); 3033 break; 3034 default: 3035 llvm_unreachable("Unexpected size"); 3036 } 3037 Inst.setOpcode(Opcode); 3038 Inst.clear(); 3039 Inst.addOperand(MCOperand::createReg(Reg)); 3040 Inst.addOperand(MCOperand::createImm(0)); 3041 } 3042 3043 void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const { 3044 Inst.setOpcode(X86::SETCCr); 3045 Inst.clear(); 3046 Inst.addOperand(MCOperand::createReg(Reg)); 3047 Inst.addOperand(MCOperand::createImm(X86::COND_O)); 3048 } 3049 3050 void createX86Lahf(MCInst &Inst) const { 3051 Inst.setOpcode(X86::LAHF); 3052 Inst.clear(); 3053 } 3054 3055 void createX86Sahf(MCInst &Inst) const { 3056 Inst.setOpcode(X86::SAHF); 3057 Inst.clear(); 3058 } 3059 3060 InstructionListType createInstrIncMemory(const MCSymbol *Target, 3061 MCContext *Ctx, 3062 bool IsLeaf) const override { 3063 InstructionListType Instrs(IsLeaf ? 13 : 11); 3064 unsigned int I = 0; 3065 3066 // Don't clobber application red zone (ABI dependent) 3067 if (IsLeaf) 3068 createStackPointerIncrement(Instrs[I++], 128, 3069 /*NoFlagsClobber=*/true); 3070 3071 // Performance improvements based on the optimization discussed at 3072 // https://reviews.llvm.org/D6629 3073 // LAHF/SAHF are used instead of PUSHF/POPF 3074 // PUSHF 3075 createPushRegister(Instrs[I++], X86::RAX, 8); 3076 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3077 createX86Lahf(Instrs[I++]); 3078 createPushRegister(Instrs[I++], X86::RAX, 8); 3079 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3080 createX86SaveOVFlagToRegister(Instrs[I++], X86::AL); 3081 // LOCK INC 3082 createIncMemory(Instrs[I++], Target, Ctx); 3083 // POPF 3084 createAddRegImm(Instrs[I++], X86::AL, 127, 1); 3085 createPopRegister(Instrs[I++], X86::RAX, 8); 3086 createX86Sahf(Instrs[I++]); 3087 createPopRegister(Instrs[I++], X86::RAX, 8); 3088 3089 if (IsLeaf) 3090 createStackPointerDecrement(Instrs[I], 128, 3091 /*NoFlagsClobber=*/true); 3092 return Instrs; 3093 } 3094 3095 void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg, 3096 int64_t Disp) const { 3097 Inst.setOpcode(X86::XCHG64rm); 3098 Inst.addOperand(MCOperand::createReg(Source)); 3099 Inst.addOperand(MCOperand::createReg(Source)); 3100 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3101 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3102 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3103 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3104 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3105 } 3106 3107 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg, 3108 int64_t Disp) const { 3109 Inst.setOpcode(X86::JMP64m); 3110 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3111 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3112 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3113 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3114 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3115 } 3116 3117 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, 3118 MCSymbol *HandlerFuncAddr, 3119 int CallSiteID, 3120 MCContext *Ctx) override { 3121 // Check if the target address expression used in the original indirect call 3122 // uses the stack pointer, which we are going to clobber. 3123 static BitVector SPAliases(getAliases(X86::RSP)); 3124 bool UsesSP = any_of(useOperands(CallInst), [&](const MCOperand &Op) { 3125 return Op.isReg() && SPAliases[Op.getReg()]; 3126 }); 3127 3128 InstructionListType Insts; 3129 MCPhysReg TempReg = getIntArgRegister(0); 3130 // Code sequence used to enter indirect call instrumentation helper: 3131 // push %rdi 3132 // add $8, %rsp ;; $rsp may be used in target, so fix it to prev val 3133 // movq target, %rdi ;; via convertIndirectCallTargetToLoad 3134 // sub $8, %rsp ;; restore correct stack value 3135 // push %rdi 3136 // movq $CallSiteID, %rdi 3137 // push %rdi 3138 // callq/jmp HandlerFuncAddr 3139 Insts.emplace_back(); 3140 createPushRegister(Insts.back(), TempReg, 8); 3141 if (UsesSP) { // Only adjust SP if we really need to 3142 Insts.emplace_back(); 3143 createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3144 } 3145 Insts.emplace_back(CallInst); 3146 // Insts.back() and CallInst now share the same annotation instruction. 3147 // Strip it from Insts.back(), only preserving tail call annotation. 3148 stripAnnotations(Insts.back(), /*KeepTC=*/true); 3149 convertIndirectCallToLoad(Insts.back(), TempReg); 3150 if (UsesSP) { 3151 Insts.emplace_back(); 3152 createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3153 } 3154 Insts.emplace_back(); 3155 createPushRegister(Insts.back(), TempReg, 8); 3156 Insts.emplace_back(); 3157 createLoadImmediate(Insts.back(), TempReg, CallSiteID); 3158 Insts.emplace_back(); 3159 createPushRegister(Insts.back(), TempReg, 8); 3160 3161 MCInst &NewCallInst = Insts.emplace_back(); 3162 createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst)); 3163 3164 // Carry over metadata including tail call marker if present. 3165 stripAnnotations(NewCallInst); 3166 moveAnnotations(std::move(CallInst), NewCallInst); 3167 3168 return Insts; 3169 } 3170 3171 InstructionListType createInstrumentedIndCallHandlerExitBB() const override { 3172 const MCPhysReg TempReg = getIntArgRegister(0); 3173 // We just need to undo the sequence created for every ind call in 3174 // instrumentIndirectTarget(), which can be accomplished minimally with: 3175 // popfq 3176 // pop %rdi 3177 // add $16, %rsp 3178 // xchg (%rsp), %rdi 3179 // jmp *-8(%rsp) 3180 InstructionListType Insts(5); 3181 createPopFlags(Insts[0], 8); 3182 createPopRegister(Insts[1], TempReg, 8); 3183 createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false); 3184 createSwap(Insts[3], TempReg, X86::RSP, 0); 3185 createIndirectBranch(Insts[4], X86::RSP, -8); 3186 return Insts; 3187 } 3188 3189 InstructionListType 3190 createInstrumentedIndTailCallHandlerExitBB() const override { 3191 const MCPhysReg TempReg = getIntArgRegister(0); 3192 // Same thing as above, but for tail calls 3193 // popfq 3194 // add $16, %rsp 3195 // pop %rdi 3196 // jmp *-16(%rsp) 3197 InstructionListType Insts(4); 3198 createPopFlags(Insts[0], 8); 3199 createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false); 3200 createPopRegister(Insts[2], TempReg, 8); 3201 createIndirectBranch(Insts[3], X86::RSP, -16); 3202 return Insts; 3203 } 3204 3205 InstructionListType 3206 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, 3207 const MCSymbol *IndCallHandler, 3208 MCContext *Ctx) override { 3209 const MCPhysReg TempReg = getIntArgRegister(0); 3210 // Code sequence used to check whether InstrTampoline was initialized 3211 // and call it if so, returns via IndCallHandler. 3212 // pushfq 3213 // mov InstrTrampoline,%rdi 3214 // cmp $0x0,%rdi 3215 // je IndCallHandler 3216 // callq *%rdi 3217 // jmpq IndCallHandler 3218 InstructionListType Insts; 3219 Insts.emplace_back(); 3220 createPushFlags(Insts.back(), 8); 3221 Insts.emplace_back(); 3222 createMove(Insts.back(), InstrTrampoline, TempReg, Ctx); 3223 InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); 3224 Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); 3225 Insts.emplace_back(); 3226 Insts.back().setOpcode(X86::CALL64r); 3227 Insts.back().addOperand(MCOperand::createReg(TempReg)); 3228 Insts.emplace_back(); 3229 createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); 3230 return Insts; 3231 } 3232 3233 InstructionListType createNumCountersGetter(MCContext *Ctx) const override { 3234 InstructionListType Insts(2); 3235 MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters"); 3236 createMove(Insts[0], NumLocs, X86::EAX, Ctx); 3237 createReturn(Insts[1]); 3238 return Insts; 3239 } 3240 3241 InstructionListType 3242 createInstrLocationsGetter(MCContext *Ctx) const override { 3243 InstructionListType Insts(2); 3244 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations"); 3245 createLea(Insts[0], Locs, X86::EAX, Ctx); 3246 createReturn(Insts[1]); 3247 return Insts; 3248 } 3249 3250 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { 3251 InstructionListType Insts(2); 3252 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables"); 3253 createLea(Insts[0], Locs, X86::EAX, Ctx); 3254 createReturn(Insts[1]); 3255 return Insts; 3256 } 3257 3258 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { 3259 InstructionListType Insts(2); 3260 MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs"); 3261 createMove(Insts[0], NumFuncs, X86::EAX, Ctx); 3262 createReturn(Insts[1]); 3263 return Insts; 3264 } 3265 3266 InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, 3267 MCContext *Ctx) const override { 3268 InstructionListType Insts(1); 3269 createUncondBranch(Insts[0], TgtSym, Ctx); 3270 return Insts; 3271 } 3272 3273 InstructionListType createDummyReturnFunction(MCContext *Ctx) const override { 3274 InstructionListType Insts(1); 3275 createReturn(Insts[0]); 3276 return Insts; 3277 } 3278 3279 BlocksVectorTy indirectCallPromotion( 3280 const MCInst &CallInst, 3281 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3282 const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms, 3283 const std::vector<MCInst *> &MethodFetchInsns, 3284 const bool MinimizeCodeSize, MCContext *Ctx) override { 3285 const bool IsTailCall = isTailCall(CallInst); 3286 const bool IsJumpTable = getJumpTable(CallInst) != 0; 3287 BlocksVectorTy Results; 3288 3289 // Label for the current code block. 3290 MCSymbol *NextTarget = nullptr; 3291 3292 // The join block which contains all the instructions following CallInst. 3293 // MergeBlock remains null if CallInst is a tail call. 3294 MCSymbol *MergeBlock = nullptr; 3295 3296 unsigned FuncAddrReg = X86::R10; 3297 3298 const bool LoadElim = !VtableSyms.empty(); 3299 assert((!LoadElim || VtableSyms.size() == Targets.size()) && 3300 "There must be a vtable entry for every method " 3301 "in the targets vector."); 3302 3303 if (MinimizeCodeSize && !LoadElim) { 3304 std::set<unsigned> UsedRegs; 3305 3306 for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) { 3307 const MCOperand &Op = CallInst.getOperand(I); 3308 if (Op.isReg()) 3309 UsedRegs.insert(Op.getReg()); 3310 } 3311 3312 if (UsedRegs.count(X86::R10) == 0) 3313 FuncAddrReg = X86::R10; 3314 else if (UsedRegs.count(X86::R11) == 0) 3315 FuncAddrReg = X86::R11; 3316 else 3317 return Results; 3318 } 3319 3320 const auto jumpToMergeBlock = [&](InstructionListType &NewCall) { 3321 assert(MergeBlock); 3322 NewCall.push_back(CallInst); 3323 MCInst &Merge = NewCall.back(); 3324 Merge.clear(); 3325 createUncondBranch(Merge, MergeBlock, Ctx); 3326 }; 3327 3328 for (unsigned int i = 0; i < Targets.size(); ++i) { 3329 Results.emplace_back(NextTarget, InstructionListType()); 3330 InstructionListType *NewCall = &Results.back().second; 3331 3332 if (MinimizeCodeSize && !LoadElim) { 3333 // Load the call target into FuncAddrReg. 3334 NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc 3335 MCInst &Target = NewCall->back(); 3336 Target.clear(); 3337 Target.setOpcode(X86::MOV64ri32); 3338 Target.addOperand(MCOperand::createReg(FuncAddrReg)); 3339 if (Targets[i].first) { 3340 // Is this OK? 3341 Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3342 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3343 } else { 3344 const uint64_t Addr = Targets[i].second; 3345 // Immediate address is out of sign extended 32 bit range. 3346 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3347 return BlocksVectorTy(); 3348 3349 Target.addOperand(MCOperand::createImm(Addr)); 3350 } 3351 3352 // Compare current call target to a specific address. 3353 NewCall->push_back(CallInst); 3354 MCInst &Compare = NewCall->back(); 3355 Compare.clear(); 3356 if (isBranchOnReg(CallInst)) 3357 Compare.setOpcode(X86::CMP64rr); 3358 else if (CallInst.getOpcode() == X86::CALL64pcrel32) 3359 Compare.setOpcode(X86::CMP64ri32); 3360 else 3361 Compare.setOpcode(X86::CMP64rm); 3362 3363 Compare.addOperand(MCOperand::createReg(FuncAddrReg)); 3364 3365 // TODO: Would be preferable to only load this value once. 3366 for (unsigned i = 0; 3367 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3368 if (!CallInst.getOperand(i).isInst()) 3369 Compare.addOperand(CallInst.getOperand(i)); 3370 } else { 3371 // Compare current call target to a specific address. 3372 NewCall->push_back(CallInst); 3373 MCInst &Compare = NewCall->back(); 3374 Compare.clear(); 3375 if (isBranchOnReg(CallInst)) 3376 Compare.setOpcode(X86::CMP64ri32); 3377 else 3378 Compare.setOpcode(X86::CMP64mi32); 3379 3380 // Original call address. 3381 for (unsigned i = 0; 3382 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3383 if (!CallInst.getOperand(i).isInst()) 3384 Compare.addOperand(CallInst.getOperand(i)); 3385 3386 // Target address. 3387 if (Targets[i].first || LoadElim) { 3388 const MCSymbol *Sym = 3389 LoadElim ? VtableSyms[i].first : Targets[i].first; 3390 const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0; 3391 const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx); 3392 if (Addend) 3393 Expr = MCBinaryExpr::createAdd( 3394 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx); 3395 Compare.addOperand(MCOperand::createExpr(Expr)); 3396 } else { 3397 const uint64_t Addr = Targets[i].second; 3398 // Immediate address is out of sign extended 32 bit range. 3399 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3400 return BlocksVectorTy(); 3401 3402 Compare.addOperand(MCOperand::createImm(Addr)); 3403 } 3404 } 3405 3406 // jump to next target compare. 3407 NextTarget = 3408 Ctx->createNamedTempSymbol(); // generate label for the next block 3409 NewCall->push_back(CallInst); 3410 3411 if (IsJumpTable) { 3412 MCInst &Je = NewCall->back(); 3413 3414 // Jump to next compare if target addresses don't match. 3415 Je.clear(); 3416 Je.setOpcode(X86::JCC_1); 3417 if (Targets[i].first) 3418 Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3419 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3420 else 3421 Je.addOperand(MCOperand::createImm(Targets[i].second)); 3422 3423 Je.addOperand(MCOperand::createImm(X86::COND_E)); 3424 assert(!isInvoke(CallInst)); 3425 } else { 3426 MCInst &Jne = NewCall->back(); 3427 3428 // Jump to next compare if target addresses don't match. 3429 Jne.clear(); 3430 Jne.setOpcode(X86::JCC_1); 3431 Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3432 NextTarget, MCSymbolRefExpr::VK_None, *Ctx))); 3433 Jne.addOperand(MCOperand::createImm(X86::COND_NE)); 3434 3435 // Call specific target directly. 3436 Results.emplace_back(Ctx->createNamedTempSymbol(), 3437 InstructionListType()); 3438 NewCall = &Results.back().second; 3439 NewCall->push_back(CallInst); 3440 MCInst &CallOrJmp = NewCall->back(); 3441 3442 CallOrJmp.clear(); 3443 3444 if (MinimizeCodeSize && !LoadElim) { 3445 CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r); 3446 CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg)); 3447 } else { 3448 CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 3449 3450 if (Targets[i].first) 3451 CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3452 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3453 else 3454 CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second)); 3455 } 3456 if (IsTailCall) 3457 setTailCall(CallOrJmp); 3458 3459 if (CallOrJmp.getOpcode() == X86::CALL64r || 3460 CallOrJmp.getOpcode() == X86::CALL64pcrel32) { 3461 if (std::optional<uint32_t> Offset = getOffset(CallInst)) 3462 // Annotated as duplicated call 3463 setOffset(CallOrJmp, *Offset); 3464 } 3465 3466 if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) { 3467 // Copy over any EH or GNU args size information from the original 3468 // call. 3469 std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst); 3470 if (EHInfo) 3471 addEHInfo(CallOrJmp, *EHInfo); 3472 int64_t GnuArgsSize = getGnuArgsSize(CallInst); 3473 if (GnuArgsSize >= 0) 3474 addGnuArgsSize(CallOrJmp, GnuArgsSize); 3475 } 3476 3477 if (!IsTailCall) { 3478 // The fallthrough block for the most common target should be 3479 // the merge block. 3480 if (i == 0) { 3481 // Fallthrough to merge block. 3482 MergeBlock = Ctx->createNamedTempSymbol(); 3483 } else { 3484 // Insert jump to the merge block if we are not doing a fallthrough. 3485 jumpToMergeBlock(*NewCall); 3486 } 3487 } 3488 } 3489 } 3490 3491 // Cold call block. 3492 Results.emplace_back(NextTarget, InstructionListType()); 3493 InstructionListType &NewCall = Results.back().second; 3494 for (const MCInst *Inst : MethodFetchInsns) 3495 if (Inst != &CallInst) 3496 NewCall.push_back(*Inst); 3497 NewCall.push_back(CallInst); 3498 3499 // Jump to merge block from cold call block 3500 if (!IsTailCall && !IsJumpTable) { 3501 jumpToMergeBlock(NewCall); 3502 3503 // Record merge block 3504 Results.emplace_back(MergeBlock, InstructionListType()); 3505 } 3506 3507 return Results; 3508 } 3509 3510 BlocksVectorTy jumpTablePromotion( 3511 const MCInst &IJmpInst, 3512 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3513 const std::vector<MCInst *> &TargetFetchInsns, 3514 MCContext *Ctx) const override { 3515 assert(getJumpTable(IJmpInst) != 0); 3516 uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg"); 3517 if (IndexReg == 0) 3518 return BlocksVectorTy(); 3519 3520 BlocksVectorTy Results; 3521 3522 // Label for the current code block. 3523 MCSymbol *NextTarget = nullptr; 3524 3525 for (unsigned int i = 0; i < Targets.size(); ++i) { 3526 Results.emplace_back(NextTarget, InstructionListType()); 3527 InstructionListType *CurBB = &Results.back().second; 3528 3529 // Compare current index to a specific index. 3530 CurBB->emplace_back(MCInst()); 3531 MCInst &CompareInst = CurBB->back(); 3532 CompareInst.setLoc(IJmpInst.getLoc()); 3533 CompareInst.setOpcode(X86::CMP64ri32); 3534 CompareInst.addOperand(MCOperand::createReg(IndexReg)); 3535 3536 const uint64_t CaseIdx = Targets[i].second; 3537 // Immediate address is out of sign extended 32 bit range. 3538 if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) 3539 return BlocksVectorTy(); 3540 3541 CompareInst.addOperand(MCOperand::createImm(CaseIdx)); 3542 shortenInstruction(CompareInst, *Ctx->getSubtargetInfo()); 3543 3544 // jump to next target compare. 3545 NextTarget = 3546 Ctx->createNamedTempSymbol(); // generate label for the next block 3547 CurBB->push_back(MCInst()); 3548 3549 MCInst &JEInst = CurBB->back(); 3550 JEInst.setLoc(IJmpInst.getLoc()); 3551 3552 // Jump to target if indices match 3553 JEInst.setOpcode(X86::JCC_1); 3554 JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3555 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3556 JEInst.addOperand(MCOperand::createImm(X86::COND_E)); 3557 } 3558 3559 // Cold call block. 3560 Results.emplace_back(NextTarget, InstructionListType()); 3561 InstructionListType &CurBB = Results.back().second; 3562 for (const MCInst *Inst : TargetFetchInsns) 3563 if (Inst != &IJmpInst) 3564 CurBB.push_back(*Inst); 3565 3566 CurBB.push_back(IJmpInst); 3567 3568 return Results; 3569 } 3570 3571 private: 3572 bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3573 MCContext *Ctx) const { 3574 Inst.setOpcode(X86::MOV64rm); 3575 Inst.addOperand(MCOperand::createReg(Reg)); 3576 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3577 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3578 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3579 Inst.addOperand(MCOperand::createExpr( 3580 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3581 *Ctx))); // Displacement 3582 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3583 3584 return true; 3585 } 3586 3587 bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3588 MCContext *Ctx) const { 3589 Inst.setOpcode(X86::LEA64r); 3590 Inst.addOperand(MCOperand::createReg(Reg)); 3591 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3592 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3593 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3594 Inst.addOperand(MCOperand::createExpr( 3595 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3596 *Ctx))); // Displacement 3597 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3598 return true; 3599 } 3600 }; 3601 3602 } // namespace 3603 3604 namespace llvm { 3605 namespace bolt { 3606 3607 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis, 3608 const MCInstrInfo *Info, 3609 const MCRegisterInfo *RegInfo) { 3610 return new X86MCPlusBuilder(Analysis, Info, RegInfo); 3611 } 3612 3613 } // namespace bolt 3614 } // namespace llvm 3615