1 //===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides X86-specific MCPlus builder. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86EncodingOptimization.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "X86MCSymbolizer.h" 17 #include "bolt/Core/MCPlus.h" 18 #include "bolt/Core/MCPlusBuilder.h" 19 #include "llvm/BinaryFormat/ELF.h" 20 #include "llvm/MC/MCContext.h" 21 #include "llvm/MC/MCFixupKindInfo.h" 22 #include "llvm/MC/MCInst.h" 23 #include "llvm/MC/MCInstBuilder.h" 24 #include "llvm/MC/MCInstrInfo.h" 25 #include "llvm/MC/MCRegister.h" 26 #include "llvm/MC/MCRegisterInfo.h" 27 #include "llvm/Support/CommandLine.h" 28 #include "llvm/Support/DataExtractor.h" 29 #include "llvm/Support/Debug.h" 30 #include "llvm/Support/Errc.h" 31 #include "llvm/Support/ErrorHandling.h" 32 #include "llvm/Support/ErrorOr.h" 33 #include <set> 34 35 #define DEBUG_TYPE "mcplus" 36 37 using namespace llvm; 38 using namespace bolt; 39 40 namespace opts { 41 42 extern cl::OptionCategory BoltOptCategory; 43 44 static cl::opt<bool> X86StripRedundantAddressSize( 45 "x86-strip-redundant-address-size", 46 cl::desc("Remove redundant Address-Size override prefix"), cl::init(true), 47 cl::cat(BoltOptCategory)); 48 49 } // namespace opts 50 51 namespace { 52 53 bool isMOVSX64rm32(const MCInst &Inst) { 54 return Inst.getOpcode() == X86::MOVSX64rm32; 55 } 56 57 bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; } 58 59 bool isADDri(const MCInst &Inst) { 60 return Inst.getOpcode() == X86::ADD64ri32 || 61 Inst.getOpcode() == X86::ADD64ri8; 62 } 63 64 #define GET_INSTRINFO_OPERAND_TYPES_ENUM 65 #define GET_INSTRINFO_OPERAND_TYPE 66 #define GET_INSTRINFO_MEM_OPERAND_SIZE 67 #include "X86GenInstrInfo.inc" 68 69 class X86MCPlusBuilder : public MCPlusBuilder { 70 public: 71 X86MCPlusBuilder(const MCInstrAnalysis *Analysis, const MCInstrInfo *Info, 72 const MCRegisterInfo *RegInfo) 73 : MCPlusBuilder(Analysis, Info, RegInfo) {} 74 75 std::unique_ptr<MCSymbolizer> 76 createTargetSymbolizer(BinaryFunction &Function, 77 bool CreateNewSymbols) const override { 78 return std::make_unique<X86MCSymbolizer>(Function, CreateNewSymbols); 79 } 80 81 bool isBranch(const MCInst &Inst) const override { 82 return Analysis->isBranch(Inst) && !isTailCall(Inst); 83 } 84 85 bool isNoop(const MCInst &Inst) const override { 86 return X86::isNOP(Inst.getOpcode()); 87 } 88 89 unsigned getCondCode(const MCInst &Inst) const override { 90 unsigned Opcode = Inst.getOpcode(); 91 if (X86::isJCC(Opcode)) 92 return Inst.getOperand(Info->get(Opcode).NumOperands - 1).getImm(); 93 return X86::COND_INVALID; 94 } 95 96 unsigned getInvertedCondCode(unsigned CC) const override { 97 switch (CC) { 98 default: return X86::COND_INVALID; 99 case X86::COND_E: return X86::COND_NE; 100 case X86::COND_NE: return X86::COND_E; 101 case X86::COND_L: return X86::COND_GE; 102 case X86::COND_LE: return X86::COND_G; 103 case X86::COND_G: return X86::COND_LE; 104 case X86::COND_GE: return X86::COND_L; 105 case X86::COND_B: return X86::COND_AE; 106 case X86::COND_BE: return X86::COND_A; 107 case X86::COND_A: return X86::COND_BE; 108 case X86::COND_AE: return X86::COND_B; 109 case X86::COND_S: return X86::COND_NS; 110 case X86::COND_NS: return X86::COND_S; 111 case X86::COND_P: return X86::COND_NP; 112 case X86::COND_NP: return X86::COND_P; 113 case X86::COND_O: return X86::COND_NO; 114 case X86::COND_NO: return X86::COND_O; 115 } 116 } 117 118 unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override { 119 enum DecodedCondCode : uint8_t { 120 DCC_EQUAL = 0x1, 121 DCC_GREATER = 0x2, 122 DCC_LESSER = 0x4, 123 DCC_GREATER_OR_LESSER = 0x6, 124 DCC_UNSIGNED = 0x8, 125 DCC_SIGNED = 0x10, 126 DCC_INVALID = 0x20, 127 }; 128 129 auto decodeCondCode = [&](unsigned CC) -> uint8_t { 130 switch (CC) { 131 default: return DCC_INVALID; 132 case X86::COND_E: return DCC_EQUAL; 133 case X86::COND_NE: return DCC_GREATER | DCC_LESSER; 134 case X86::COND_L: return DCC_LESSER | DCC_SIGNED; 135 case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED; 136 case X86::COND_G: return DCC_GREATER | DCC_SIGNED; 137 case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED; 138 case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED; 139 case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED; 140 case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED; 141 case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED; 142 } 143 }; 144 145 uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2); 146 147 if (DCC & DCC_INVALID) 148 return X86::COND_INVALID; 149 150 if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED) 151 return X86::COND_INVALID; 152 153 switch (DCC) { 154 default: return X86::COND_INVALID; 155 case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE; 156 case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE; 157 case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE; 158 case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE; 159 case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE; 160 case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE; 161 case DCC_GREATER | DCC_LESSER: return X86::COND_NE; 162 case DCC_EQUAL | DCC_SIGNED: return X86::COND_E; 163 case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E; 164 case DCC_EQUAL: return X86::COND_E; 165 case DCC_LESSER | DCC_SIGNED: return X86::COND_L; 166 case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B; 167 case DCC_GREATER | DCC_SIGNED: return X86::COND_G; 168 case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A; 169 } 170 } 171 172 bool isValidCondCode(unsigned CC) const override { 173 return (CC != X86::COND_INVALID); 174 } 175 176 bool isBreakpoint(const MCInst &Inst) const override { 177 return Inst.getOpcode() == X86::INT3; 178 } 179 180 bool isPrefix(const MCInst &Inst) const override { 181 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 182 return X86II::isPrefix(Desc.TSFlags); 183 } 184 185 bool isRep(const MCInst &Inst) const override { 186 return Inst.getFlags() == X86::IP_HAS_REPEAT; 187 } 188 189 bool deleteREPPrefix(MCInst &Inst) const override { 190 if (Inst.getFlags() == X86::IP_HAS_REPEAT) { 191 Inst.setFlags(0); 192 return true; 193 } 194 return false; 195 } 196 197 // FIXME: For compatibility with old LLVM only! 198 bool isTerminator(const MCInst &Inst) const override { 199 unsigned Opcode = Inst.getOpcode(); 200 return Info->get(Opcode).isTerminator() || X86::isUD1(Opcode) || 201 X86::isUD2(Opcode); 202 } 203 204 bool isIndirectCall(const MCInst &Inst) const override { 205 return isCall(Inst) && 206 ((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(0).isReg()); 207 } 208 209 bool isPop(const MCInst &Inst) const override { 210 return getPopSize(Inst) == 0 ? false : true; 211 } 212 213 bool isTerminateBranch(const MCInst &Inst) const override { 214 return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64; 215 } 216 217 int getPopSize(const MCInst &Inst) const override { 218 switch (Inst.getOpcode()) { 219 case X86::POP16r: 220 case X86::POP16rmm: 221 case X86::POP16rmr: 222 case X86::POPF16: 223 case X86::POPA16: 224 case X86::POPDS16: 225 case X86::POPES16: 226 case X86::POPFS16: 227 case X86::POPGS16: 228 case X86::POPSS16: 229 return 2; 230 case X86::POP32r: 231 case X86::POP32rmm: 232 case X86::POP32rmr: 233 case X86::POPA32: 234 case X86::POPDS32: 235 case X86::POPES32: 236 case X86::POPF32: 237 case X86::POPFS32: 238 case X86::POPGS32: 239 case X86::POPSS32: 240 return 4; 241 case X86::POP64r: 242 case X86::POP64rmm: 243 case X86::POP64rmr: 244 case X86::POPF64: 245 case X86::POPFS64: 246 case X86::POPGS64: 247 return 8; 248 } 249 return 0; 250 } 251 252 bool isPush(const MCInst &Inst) const override { 253 return getPushSize(Inst) == 0 ? false : true; 254 } 255 256 int getPushSize(const MCInst &Inst) const override { 257 switch (Inst.getOpcode()) { 258 case X86::PUSH16i8: 259 case X86::PUSH16r: 260 case X86::PUSH16rmm: 261 case X86::PUSH16rmr: 262 case X86::PUSHA16: 263 case X86::PUSHCS16: 264 case X86::PUSHDS16: 265 case X86::PUSHES16: 266 case X86::PUSHF16: 267 case X86::PUSHFS16: 268 case X86::PUSHGS16: 269 case X86::PUSHSS16: 270 case X86::PUSH16i: 271 return 2; 272 case X86::PUSH32i8: 273 case X86::PUSH32r: 274 case X86::PUSH32rmm: 275 case X86::PUSH32rmr: 276 case X86::PUSHA32: 277 case X86::PUSHCS32: 278 case X86::PUSHDS32: 279 case X86::PUSHES32: 280 case X86::PUSHF32: 281 case X86::PUSHFS32: 282 case X86::PUSHGS32: 283 case X86::PUSHSS32: 284 case X86::PUSH32i: 285 return 4; 286 case X86::PUSH64i32: 287 case X86::PUSH64i8: 288 case X86::PUSH64r: 289 case X86::PUSH64rmm: 290 case X86::PUSH64rmr: 291 case X86::PUSHF64: 292 case X86::PUSHFS64: 293 case X86::PUSHGS64: 294 return 8; 295 } 296 return 0; 297 } 298 299 bool isSUB(const MCInst &Inst) const override { 300 return X86::isSUB(Inst.getOpcode()); 301 } 302 303 bool isLEA64r(const MCInst &Inst) const override { 304 return Inst.getOpcode() == X86::LEA64r; 305 } 306 307 bool isLeave(const MCInst &Inst) const override { 308 return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64; 309 } 310 311 bool isMoveMem2Reg(const MCInst &Inst) const override { 312 switch (Inst.getOpcode()) { 313 case X86::MOV16rm: 314 case X86::MOV32rm: 315 case X86::MOV64rm: 316 return true; 317 } 318 return false; 319 } 320 321 bool isUnsupportedBranch(const MCInst &Inst) const override { 322 switch (Inst.getOpcode()) { 323 default: 324 return false; 325 case X86::LOOP: 326 case X86::LOOPE: 327 case X86::LOOPNE: 328 case X86::JECXZ: 329 case X86::JRCXZ: 330 return true; 331 } 332 } 333 334 bool isLoad(const MCInst &Inst) const override { 335 if (isPop(Inst)) 336 return true; 337 338 int MemOpNo = getMemoryOperandNo(Inst); 339 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 340 341 if (MemOpNo == -1) 342 return false; 343 344 return MCII.mayLoad(); 345 } 346 347 bool isStore(const MCInst &Inst) const override { 348 if (isPush(Inst)) 349 return true; 350 351 int MemOpNo = getMemoryOperandNo(Inst); 352 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 353 354 if (MemOpNo == -1) 355 return false; 356 357 return MCII.mayStore(); 358 } 359 360 bool isCleanRegXOR(const MCInst &Inst) const override { 361 switch (Inst.getOpcode()) { 362 case X86::XOR16rr: 363 case X86::XOR32rr: 364 case X86::XOR64rr: 365 break; 366 default: 367 return false; 368 } 369 return (Inst.getOperand(0).getReg() == Inst.getOperand(2).getReg()); 370 } 371 372 bool isPacked(const MCInst &Inst) const override { 373 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 374 return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD; 375 } 376 377 bool shouldRecordCodeRelocation(uint64_t RelType) const override { 378 switch (RelType) { 379 case ELF::R_X86_64_8: 380 case ELF::R_X86_64_16: 381 case ELF::R_X86_64_32: 382 case ELF::R_X86_64_32S: 383 case ELF::R_X86_64_64: 384 case ELF::R_X86_64_PC8: 385 case ELF::R_X86_64_PC32: 386 case ELF::R_X86_64_PC64: 387 case ELF::R_X86_64_GOTPCRELX: 388 case ELF::R_X86_64_REX_GOTPCRELX: 389 return true; 390 case ELF::R_X86_64_PLT32: 391 case ELF::R_X86_64_GOTPCREL: 392 case ELF::R_X86_64_TPOFF32: 393 case ELF::R_X86_64_GOTTPOFF: 394 return false; 395 default: 396 llvm_unreachable("Unexpected x86 relocation type in code"); 397 } 398 } 399 400 unsigned getTrapFillValue() const override { return 0xCC; } 401 402 struct IndJmpMatcherFrag1 : MCInstMatcher { 403 std::unique_ptr<MCInstMatcher> Base; 404 std::unique_ptr<MCInstMatcher> Scale; 405 std::unique_ptr<MCInstMatcher> Index; 406 std::unique_ptr<MCInstMatcher> Offset; 407 408 IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 409 std::unique_ptr<MCInstMatcher> Scale, 410 std::unique_ptr<MCInstMatcher> Index, 411 std::unique_ptr<MCInstMatcher> Offset) 412 : Base(std::move(Base)), Scale(std::move(Scale)), 413 Index(std::move(Index)), Offset(std::move(Offset)) {} 414 415 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 416 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 417 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 418 return false; 419 420 if (CurInst->getOpcode() != X86::JMP64m) 421 return false; 422 423 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 424 if (MemOpNo == -1) 425 return false; 426 427 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 428 return false; 429 if (!Scale->match(MRI, MIB, this->InstrWindow, 430 MemOpNo + X86::AddrScaleAmt)) 431 return false; 432 if (!Index->match(MRI, MIB, this->InstrWindow, 433 MemOpNo + X86::AddrIndexReg)) 434 return false; 435 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 436 return false; 437 return true; 438 } 439 440 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 441 MIB.addAnnotation(*CurInst, Annotation, true); 442 Base->annotate(MIB, Annotation); 443 Scale->annotate(MIB, Annotation); 444 Index->annotate(MIB, Annotation); 445 Offset->annotate(MIB, Annotation); 446 } 447 }; 448 449 std::unique_ptr<MCInstMatcher> 450 matchIndJmp(std::unique_ptr<MCInstMatcher> Base, 451 std::unique_ptr<MCInstMatcher> Scale, 452 std::unique_ptr<MCInstMatcher> Index, 453 std::unique_ptr<MCInstMatcher> Offset) const override { 454 return std::unique_ptr<MCInstMatcher>( 455 new IndJmpMatcherFrag1(std::move(Base), std::move(Scale), 456 std::move(Index), std::move(Offset))); 457 } 458 459 struct IndJmpMatcherFrag2 : MCInstMatcher { 460 std::unique_ptr<MCInstMatcher> Reg; 461 462 IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg) 463 : Reg(std::move(Reg)) {} 464 465 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 466 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 467 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 468 return false; 469 470 if (CurInst->getOpcode() != X86::JMP64r) 471 return false; 472 473 return Reg->match(MRI, MIB, this->InstrWindow, 0); 474 } 475 476 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 477 MIB.addAnnotation(*CurInst, Annotation, true); 478 Reg->annotate(MIB, Annotation); 479 } 480 }; 481 482 std::unique_ptr<MCInstMatcher> 483 matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override { 484 return std::unique_ptr<MCInstMatcher>( 485 new IndJmpMatcherFrag2(std::move(Target))); 486 } 487 488 struct LoadMatcherFrag1 : MCInstMatcher { 489 std::unique_ptr<MCInstMatcher> Base; 490 std::unique_ptr<MCInstMatcher> Scale; 491 std::unique_ptr<MCInstMatcher> Index; 492 std::unique_ptr<MCInstMatcher> Offset; 493 494 LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base, 495 std::unique_ptr<MCInstMatcher> Scale, 496 std::unique_ptr<MCInstMatcher> Index, 497 std::unique_ptr<MCInstMatcher> Offset) 498 : Base(std::move(Base)), Scale(std::move(Scale)), 499 Index(std::move(Index)), Offset(std::move(Offset)) {} 500 501 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 502 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 503 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 504 return false; 505 506 if (CurInst->getOpcode() != X86::MOV64rm && 507 CurInst->getOpcode() != X86::MOVSX64rm32) 508 return false; 509 510 int MemOpNo = MIB.getMemoryOperandNo(*CurInst); 511 if (MemOpNo == -1) 512 return false; 513 514 if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg)) 515 return false; 516 if (!Scale->match(MRI, MIB, this->InstrWindow, 517 MemOpNo + X86::AddrScaleAmt)) 518 return false; 519 if (!Index->match(MRI, MIB, this->InstrWindow, 520 MemOpNo + X86::AddrIndexReg)) 521 return false; 522 if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp)) 523 return false; 524 return true; 525 } 526 527 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 528 MIB.addAnnotation(*CurInst, Annotation, true); 529 Base->annotate(MIB, Annotation); 530 Scale->annotate(MIB, Annotation); 531 Index->annotate(MIB, Annotation); 532 Offset->annotate(MIB, Annotation); 533 } 534 }; 535 536 std::unique_ptr<MCInstMatcher> 537 matchLoad(std::unique_ptr<MCInstMatcher> Base, 538 std::unique_ptr<MCInstMatcher> Scale, 539 std::unique_ptr<MCInstMatcher> Index, 540 std::unique_ptr<MCInstMatcher> Offset) const override { 541 return std::unique_ptr<MCInstMatcher>( 542 new LoadMatcherFrag1(std::move(Base), std::move(Scale), 543 std::move(Index), std::move(Offset))); 544 } 545 546 struct AddMatcher : MCInstMatcher { 547 std::unique_ptr<MCInstMatcher> A; 548 std::unique_ptr<MCInstMatcher> B; 549 550 AddMatcher(std::unique_ptr<MCInstMatcher> A, 551 std::unique_ptr<MCInstMatcher> B) 552 : A(std::move(A)), B(std::move(B)) {} 553 554 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 555 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 556 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 557 return false; 558 559 if (CurInst->getOpcode() == X86::ADD64rr || 560 CurInst->getOpcode() == X86::ADD64rr_DB || 561 CurInst->getOpcode() == X86::ADD64rr_REV) { 562 if (!A->match(MRI, MIB, this->InstrWindow, 1)) { 563 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 564 return false; 565 return A->match(MRI, MIB, this->InstrWindow, 2); 566 } 567 568 if (B->match(MRI, MIB, this->InstrWindow, 2)) 569 return true; 570 571 if (!B->match(MRI, MIB, this->InstrWindow, 1)) 572 return false; 573 return A->match(MRI, MIB, this->InstrWindow, 2); 574 } 575 576 return false; 577 } 578 579 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 580 MIB.addAnnotation(*CurInst, Annotation, true); 581 A->annotate(MIB, Annotation); 582 B->annotate(MIB, Annotation); 583 } 584 }; 585 586 std::unique_ptr<MCInstMatcher> 587 matchAdd(std::unique_ptr<MCInstMatcher> A, 588 std::unique_ptr<MCInstMatcher> B) const override { 589 return std::unique_ptr<MCInstMatcher>( 590 new AddMatcher(std::move(A), std::move(B))); 591 } 592 593 struct LEAMatcher : MCInstMatcher { 594 std::unique_ptr<MCInstMatcher> Target; 595 596 LEAMatcher(std::unique_ptr<MCInstMatcher> Target) 597 : Target(std::move(Target)) {} 598 599 bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB, 600 MutableArrayRef<MCInst> InInstrWindow, int OpNum) override { 601 if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum)) 602 return false; 603 604 if (CurInst->getOpcode() != X86::LEA64r) 605 return false; 606 607 if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 || 608 CurInst->getOperand(1 + X86::AddrIndexReg).getReg() != 609 X86::NoRegister || 610 (CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != 611 X86::NoRegister && 612 CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP)) 613 return false; 614 615 return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp); 616 } 617 618 void annotate(MCPlusBuilder &MIB, StringRef Annotation) override { 619 MIB.addAnnotation(*CurInst, Annotation, true); 620 Target->annotate(MIB, Annotation); 621 } 622 }; 623 624 std::unique_ptr<MCInstMatcher> 625 matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override { 626 return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target))); 627 } 628 629 bool hasPCRelOperand(const MCInst &Inst) const override { 630 for (const MCOperand &Operand : Inst) 631 if (Operand.isReg() && Operand.getReg() == X86::RIP) 632 return true; 633 return false; 634 } 635 636 int getMemoryOperandNo(const MCInst &Inst) const override { 637 unsigned Opcode = Inst.getOpcode(); 638 const MCInstrDesc &Desc = Info->get(Opcode); 639 int MemOpNo = X86II::getMemoryOperandNo(Desc.TSFlags); 640 if (MemOpNo >= 0) 641 MemOpNo += X86II::getOperandBias(Desc); 642 return MemOpNo; 643 } 644 645 bool hasEVEXEncoding(const MCInst &Inst) const override { 646 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 647 return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX; 648 } 649 650 bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override { 651 const auto *I = Insts.begin(); 652 while (I != Insts.end() && isPrefix(*I)) 653 ++I; 654 if (I == Insts.end()) 655 return false; 656 657 const MCInst &FirstInst = *I; 658 ++I; 659 while (I != Insts.end() && isPrefix(*I)) 660 ++I; 661 if (I == Insts.end()) 662 return false; 663 const MCInst &SecondInst = *I; 664 665 if (!isConditionalBranch(SecondInst)) 666 return false; 667 // Cannot fuse if the first instruction uses RIP-relative memory. 668 if (hasPCRelOperand(FirstInst)) 669 return false; 670 671 const X86::FirstMacroFusionInstKind CmpKind = 672 X86::classifyFirstOpcodeInMacroFusion(FirstInst.getOpcode()); 673 if (CmpKind == X86::FirstMacroFusionInstKind::Invalid) 674 return false; 675 676 X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(SecondInst)); 677 X86::SecondMacroFusionInstKind BranchKind = 678 X86::classifySecondCondCodeInMacroFusion(CC); 679 if (BranchKind == X86::SecondMacroFusionInstKind::Invalid) 680 return false; 681 return X86::isMacroFused(CmpKind, BranchKind); 682 } 683 684 std::optional<X86MemOperand> 685 evaluateX86MemoryOperand(const MCInst &Inst) const override { 686 int MemOpNo = getMemoryOperandNo(Inst); 687 if (MemOpNo < 0) 688 return std::nullopt; 689 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 690 691 if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst)) 692 return std::nullopt; 693 694 const MCOperand &Base = Inst.getOperand(MemOpOffset + X86::AddrBaseReg); 695 const MCOperand &Scale = Inst.getOperand(MemOpOffset + X86::AddrScaleAmt); 696 const MCOperand &Index = Inst.getOperand(MemOpOffset + X86::AddrIndexReg); 697 const MCOperand &Disp = Inst.getOperand(MemOpOffset + X86::AddrDisp); 698 const MCOperand &Segment = 699 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg); 700 701 // Make sure it is a well-formed memory operand. 702 if (!Base.isReg() || !Scale.isImm() || !Index.isReg() || 703 (!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg()) 704 return std::nullopt; 705 706 X86MemOperand MO; 707 MO.BaseRegNum = Base.getReg(); 708 MO.ScaleImm = Scale.getImm(); 709 MO.IndexRegNum = Index.getReg(); 710 MO.DispImm = Disp.isImm() ? Disp.getImm() : 0; 711 MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr; 712 MO.SegRegNum = Segment.getReg(); 713 return MO; 714 } 715 716 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target, 717 uint64_t Address, 718 uint64_t Size) const override { 719 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 720 if (!MO) 721 return false; 722 723 // Make sure it's a well-formed addressing we can statically evaluate. 724 if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) || 725 MO->IndexRegNum != X86::NoRegister || 726 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 727 return false; 728 729 Target = MO->DispImm; 730 if (MO->BaseRegNum == X86::RIP) { 731 assert(Size != 0 && "instruction size required in order to statically " 732 "evaluate RIP-relative address"); 733 Target += Address + Size; 734 } 735 return true; 736 } 737 738 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override { 739 int MemOpNo = getMemoryOperandNo(Inst); 740 if (MemOpNo < 0) 741 return Inst.end(); 742 return Inst.begin() + (MemOpNo + X86::AddrDisp); 743 } 744 745 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override { 746 MCOperand *OI = getMemOperandDisp(Inst); 747 if (OI == Inst.end()) 748 return false; 749 *OI = Operand; 750 return true; 751 } 752 753 /// Get the registers used as function parameters. 754 /// This function is specific to the x86_64 abi on Linux. 755 BitVector getRegsUsedAsParams() const override { 756 BitVector Regs = BitVector(RegInfo->getNumRegs(), false); 757 Regs |= getAliases(X86::RSI); 758 Regs |= getAliases(X86::RDI); 759 Regs |= getAliases(X86::RDX); 760 Regs |= getAliases(X86::RCX); 761 Regs |= getAliases(X86::R8); 762 Regs |= getAliases(X86::R9); 763 return Regs; 764 } 765 766 void getCalleeSavedRegs(BitVector &Regs) const override { 767 Regs |= getAliases(X86::RBX); 768 Regs |= getAliases(X86::RBP); 769 Regs |= getAliases(X86::R12); 770 Regs |= getAliases(X86::R13); 771 Regs |= getAliases(X86::R14); 772 Regs |= getAliases(X86::R15); 773 } 774 775 void getDefaultDefIn(BitVector &Regs) const override { 776 assert(Regs.size() >= RegInfo->getNumRegs() && 777 "The size of BitVector is less than RegInfo->getNumRegs()."); 778 Regs.set(X86::RAX); 779 Regs.set(X86::RCX); 780 Regs.set(X86::RDX); 781 Regs.set(X86::RSI); 782 Regs.set(X86::RDI); 783 Regs.set(X86::R8); 784 Regs.set(X86::R9); 785 Regs.set(X86::XMM0); 786 Regs.set(X86::XMM1); 787 Regs.set(X86::XMM2); 788 Regs.set(X86::XMM3); 789 Regs.set(X86::XMM4); 790 Regs.set(X86::XMM5); 791 Regs.set(X86::XMM6); 792 Regs.set(X86::XMM7); 793 } 794 795 void getDefaultLiveOut(BitVector &Regs) const override { 796 assert(Regs.size() >= RegInfo->getNumRegs() && 797 "The size of BitVector is less than RegInfo->getNumRegs()."); 798 Regs |= getAliases(X86::RAX); 799 Regs |= getAliases(X86::RDX); 800 Regs |= getAliases(X86::RCX); 801 Regs |= getAliases(X86::XMM0); 802 Regs |= getAliases(X86::XMM1); 803 } 804 805 void getGPRegs(BitVector &Regs, bool IncludeAlias) const override { 806 if (IncludeAlias) { 807 Regs |= getAliases(X86::RAX); 808 Regs |= getAliases(X86::RBX); 809 Regs |= getAliases(X86::RBP); 810 Regs |= getAliases(X86::RSI); 811 Regs |= getAliases(X86::RDI); 812 Regs |= getAliases(X86::RDX); 813 Regs |= getAliases(X86::RCX); 814 Regs |= getAliases(X86::R8); 815 Regs |= getAliases(X86::R9); 816 Regs |= getAliases(X86::R10); 817 Regs |= getAliases(X86::R11); 818 Regs |= getAliases(X86::R12); 819 Regs |= getAliases(X86::R13); 820 Regs |= getAliases(X86::R14); 821 Regs |= getAliases(X86::R15); 822 return; 823 } 824 Regs.set(X86::RAX); 825 Regs.set(X86::RBX); 826 Regs.set(X86::RBP); 827 Regs.set(X86::RSI); 828 Regs.set(X86::RDI); 829 Regs.set(X86::RDX); 830 Regs.set(X86::RCX); 831 Regs.set(X86::R8); 832 Regs.set(X86::R9); 833 Regs.set(X86::R10); 834 Regs.set(X86::R11); 835 Regs.set(X86::R12); 836 Regs.set(X86::R13); 837 Regs.set(X86::R14); 838 Regs.set(X86::R15); 839 } 840 841 void getClassicGPRegs(BitVector &Regs) const override { 842 Regs |= getAliases(X86::RAX); 843 Regs |= getAliases(X86::RBX); 844 Regs |= getAliases(X86::RBP); 845 Regs |= getAliases(X86::RSI); 846 Regs |= getAliases(X86::RDI); 847 Regs |= getAliases(X86::RDX); 848 Regs |= getAliases(X86::RCX); 849 } 850 851 void getRepRegs(BitVector &Regs) const override { 852 Regs |= getAliases(X86::RCX); 853 } 854 855 MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override { 856 Reg = getX86SubSuperRegister(Reg, Size * 8); 857 assert((Reg != X86::NoRegister) && "Invalid register"); 858 return Reg; 859 } 860 861 bool isUpper8BitReg(MCPhysReg Reg) const override { 862 switch (Reg) { 863 case X86::AH: 864 case X86::BH: 865 case X86::CH: 866 case X86::DH: 867 return true; 868 default: 869 return false; 870 } 871 } 872 873 bool cannotUseREX(const MCInst &Inst) const override { 874 switch (Inst.getOpcode()) { 875 case X86::MOV8mr_NOREX: 876 case X86::MOV8rm_NOREX: 877 case X86::MOV8rr_NOREX: 878 case X86::MOVSX32rm8_NOREX: 879 case X86::MOVSX32rr8_NOREX: 880 case X86::MOVZX32rm8_NOREX: 881 case X86::MOVZX32rr8_NOREX: 882 case X86::MOV8mr: 883 case X86::MOV8rm: 884 case X86::MOV8rr: 885 case X86::MOVSX32rm8: 886 case X86::MOVSX32rr8: 887 case X86::MOVZX32rm8: 888 case X86::MOVZX32rr8: 889 case X86::TEST8ri: 890 for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) { 891 if (!Operand.isReg()) 892 continue; 893 if (isUpper8BitReg(Operand.getReg())) 894 return true; 895 } 896 [[fallthrough]]; 897 default: 898 return false; 899 } 900 } 901 902 static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) { 903 using namespace llvm::X86; 904 int OpType = getOperandType(Inst.getOpcode(), MemOpNo); 905 return getMemOperandSize(OpType) / 8; 906 } 907 908 /// Classifying a stack access as *not* "SIMPLE" here means we don't know how 909 /// to change this instruction memory access. It will disable any changes to 910 /// the stack layout, so we can't do the most aggressive form of shrink 911 /// wrapping. We must do so in a way that keeps the original stack layout. 912 /// Otherwise you need to adjust the offset of all instructions accessing the 913 /// stack: we can't do that anymore because there is one instruction that is 914 /// not simple. There are other implications as well. We have heuristics to 915 /// detect when a register is callee-saved and thus eligible for shrink 916 /// wrapping. If you are restoring a register using a non-simple stack access, 917 /// then it is classified as NOT callee-saved, and it disables shrink wrapping 918 /// for *that* register (but not for others). 919 /// 920 /// Classifying a stack access as "size 0" or detecting an indexed memory 921 /// access (to address a vector, for example) here means we know there is a 922 /// stack access, but we can't quite understand how wide is the access in 923 /// bytes. This is very serious because we can't understand how memory 924 /// accesses alias with each other for this function. This will essentially 925 /// disable not only shrink wrapping but all frame analysis, it will fail it 926 /// as "we don't understand this function and we give up on it". 927 bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore, 928 bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm, 929 uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size, 930 bool &IsSimple, bool &IsIndexed) const override { 931 // Detect simple push/pop cases first 932 if (int Sz = getPushSize(Inst)) { 933 IsLoad = false; 934 IsStore = true; 935 IsStoreFromReg = true; 936 StackPtrReg = X86::RSP; 937 StackOffset = -Sz; 938 Size = Sz; 939 IsSimple = true; 940 if (Inst.getOperand(0).isImm()) 941 SrcImm = Inst.getOperand(0).getImm(); 942 else if (Inst.getOperand(0).isReg()) 943 Reg = Inst.getOperand(0).getReg(); 944 else 945 IsSimple = false; 946 947 return true; 948 } 949 if (int Sz = getPopSize(Inst)) { 950 IsLoad = true; 951 IsStore = false; 952 if (Inst.getNumOperands() == 0 || !Inst.getOperand(0).isReg()) { 953 IsSimple = false; 954 } else { 955 Reg = Inst.getOperand(0).getReg(); 956 IsSimple = true; 957 } 958 StackPtrReg = X86::RSP; 959 StackOffset = 0; 960 Size = Sz; 961 return true; 962 } 963 964 struct InstInfo { 965 // Size in bytes that Inst loads from memory. 966 uint8_t DataSize; 967 bool IsLoad; 968 bool IsStore; 969 bool StoreFromReg; 970 bool Simple; 971 }; 972 973 InstInfo I; 974 int MemOpNo = getMemoryOperandNo(Inst); 975 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 976 // If it is not dealing with a memory operand, we discard it 977 if (MemOpNo == -1 || MCII.isCall()) 978 return false; 979 980 switch (Inst.getOpcode()) { 981 default: { 982 bool IsLoad = MCII.mayLoad(); 983 bool IsStore = MCII.mayStore(); 984 // Is it LEA? (deals with memory but is not loading nor storing) 985 if (!IsLoad && !IsStore) { 986 I = {0, IsLoad, IsStore, false, false}; 987 break; 988 } 989 uint8_t Sz = getMemDataSize(Inst, MemOpNo); 990 I = {Sz, IsLoad, IsStore, false, false}; 991 break; 992 } 993 // Report simple stack accesses 994 case X86::MOV8rm: I = {1, true, false, false, true}; break; 995 case X86::MOV16rm: I = {2, true, false, false, true}; break; 996 case X86::MOV32rm: I = {4, true, false, false, true}; break; 997 case X86::MOV64rm: I = {8, true, false, false, true}; break; 998 case X86::MOV8mr: I = {1, false, true, true, true}; break; 999 case X86::MOV16mr: I = {2, false, true, true, true}; break; 1000 case X86::MOV32mr: I = {4, false, true, true, true}; break; 1001 case X86::MOV64mr: I = {8, false, true, true, true}; break; 1002 case X86::MOV8mi: I = {1, false, true, false, true}; break; 1003 case X86::MOV16mi: I = {2, false, true, false, true}; break; 1004 case X86::MOV32mi: I = {4, false, true, false, true}; break; 1005 } // end switch (Inst.getOpcode()) 1006 1007 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1008 if (!MO) { 1009 LLVM_DEBUG(dbgs() << "Evaluate failed on "); 1010 LLVM_DEBUG(Inst.dump()); 1011 return false; 1012 } 1013 1014 // Make sure it's a stack access 1015 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) 1016 return false; 1017 1018 IsLoad = I.IsLoad; 1019 IsStore = I.IsStore; 1020 IsStoreFromReg = I.StoreFromReg; 1021 Size = I.DataSize; 1022 IsSimple = I.Simple; 1023 StackPtrReg = MO->BaseRegNum; 1024 StackOffset = MO->DispImm; 1025 IsIndexed = 1026 MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister; 1027 1028 if (!I.Simple) 1029 return true; 1030 1031 // Retrieve related register in simple MOV from/to stack operations. 1032 unsigned MemOpOffset = static_cast<unsigned>(MemOpNo); 1033 if (I.IsLoad) { 1034 MCOperand RegOpnd = Inst.getOperand(0); 1035 assert(RegOpnd.isReg() && "unexpected destination operand"); 1036 Reg = RegOpnd.getReg(); 1037 } else if (I.IsStore) { 1038 MCOperand SrcOpnd = 1039 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1040 if (I.StoreFromReg) { 1041 assert(SrcOpnd.isReg() && "unexpected source operand"); 1042 Reg = SrcOpnd.getReg(); 1043 } else { 1044 assert(SrcOpnd.isImm() && "unexpected source operand"); 1045 SrcImm = SrcOpnd.getImm(); 1046 } 1047 } 1048 1049 return true; 1050 } 1051 1052 void changeToPushOrPop(MCInst &Inst) const override { 1053 assert(!isPush(Inst) && !isPop(Inst)); 1054 1055 struct InstInfo { 1056 // Size in bytes that Inst loads from memory. 1057 uint8_t DataSize; 1058 bool IsLoad; 1059 bool StoreFromReg; 1060 }; 1061 1062 InstInfo I; 1063 switch (Inst.getOpcode()) { 1064 default: { 1065 llvm_unreachable("Unhandled opcode"); 1066 return; 1067 } 1068 case X86::MOV16rm: I = {2, true, false}; break; 1069 case X86::MOV32rm: I = {4, true, false}; break; 1070 case X86::MOV64rm: I = {8, true, false}; break; 1071 case X86::MOV16mr: I = {2, false, true}; break; 1072 case X86::MOV32mr: I = {4, false, true}; break; 1073 case X86::MOV64mr: I = {8, false, true}; break; 1074 case X86::MOV16mi: I = {2, false, false}; break; 1075 case X86::MOV32mi: I = {4, false, false}; break; 1076 } // end switch (Inst.getOpcode()) 1077 1078 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1079 if (!MO) { 1080 llvm_unreachable("Evaluate failed"); 1081 return; 1082 } 1083 // Make sure it's a stack access 1084 if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) { 1085 llvm_unreachable("Not a stack access"); 1086 return; 1087 } 1088 1089 unsigned MemOpOffset = getMemoryOperandNo(Inst); 1090 unsigned NewOpcode = 0; 1091 if (I.IsLoad) { 1092 switch (I.DataSize) { 1093 case 2: NewOpcode = X86::POP16r; break; 1094 case 4: NewOpcode = X86::POP32r; break; 1095 case 8: NewOpcode = X86::POP64r; break; 1096 default: 1097 llvm_unreachable("Unexpected size"); 1098 } 1099 unsigned RegOpndNum = Inst.getOperand(0).getReg(); 1100 Inst.clear(); 1101 Inst.setOpcode(NewOpcode); 1102 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1103 } else { 1104 MCOperand SrcOpnd = 1105 Inst.getOperand(MemOpOffset + X86::AddrSegmentReg + 1); 1106 if (I.StoreFromReg) { 1107 switch (I.DataSize) { 1108 case 2: NewOpcode = X86::PUSH16r; break; 1109 case 4: NewOpcode = X86::PUSH32r; break; 1110 case 8: NewOpcode = X86::PUSH64r; break; 1111 default: 1112 llvm_unreachable("Unexpected size"); 1113 } 1114 assert(SrcOpnd.isReg() && "Unexpected source operand"); 1115 unsigned RegOpndNum = SrcOpnd.getReg(); 1116 Inst.clear(); 1117 Inst.setOpcode(NewOpcode); 1118 Inst.addOperand(MCOperand::createReg(RegOpndNum)); 1119 } else { 1120 switch (I.DataSize) { 1121 case 2: NewOpcode = X86::PUSH16i8; break; 1122 case 4: NewOpcode = X86::PUSH32i8; break; 1123 case 8: NewOpcode = X86::PUSH64i32; break; 1124 default: 1125 llvm_unreachable("Unexpected size"); 1126 } 1127 assert(SrcOpnd.isImm() && "Unexpected source operand"); 1128 int64_t SrcImm = SrcOpnd.getImm(); 1129 Inst.clear(); 1130 Inst.setOpcode(NewOpcode); 1131 Inst.addOperand(MCOperand::createImm(SrcImm)); 1132 } 1133 } 1134 } 1135 1136 bool isStackAdjustment(const MCInst &Inst) const override { 1137 switch (Inst.getOpcode()) { 1138 default: 1139 return false; 1140 case X86::SUB64ri32: 1141 case X86::SUB64ri8: 1142 case X86::ADD64ri32: 1143 case X86::ADD64ri8: 1144 case X86::LEA64r: 1145 break; 1146 } 1147 1148 return any_of(defOperands(Inst), [](const MCOperand &Op) { 1149 return Op.isReg() && Op.getReg() == X86::RSP; 1150 }); 1151 } 1152 1153 bool 1154 evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output, 1155 std::pair<MCPhysReg, int64_t> Input1, 1156 std::pair<MCPhysReg, int64_t> Input2) const override { 1157 1158 auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> { 1159 if (Reg == Input1.first) 1160 return Input1.second; 1161 if (Reg == Input2.first) 1162 return Input2.second; 1163 return make_error_code(errc::result_out_of_range); 1164 }; 1165 1166 switch (Inst.getOpcode()) { 1167 default: 1168 return false; 1169 1170 case X86::SUB64ri32: 1171 case X86::SUB64ri8: 1172 if (!Inst.getOperand(2).isImm()) 1173 return false; 1174 if (ErrorOr<int64_t> InputVal = 1175 getOperandVal(Inst.getOperand(1).getReg())) 1176 Output = *InputVal - Inst.getOperand(2).getImm(); 1177 else 1178 return false; 1179 break; 1180 case X86::ADD64ri32: 1181 case X86::ADD64ri8: 1182 if (!Inst.getOperand(2).isImm()) 1183 return false; 1184 if (ErrorOr<int64_t> InputVal = 1185 getOperandVal(Inst.getOperand(1).getReg())) 1186 Output = *InputVal + Inst.getOperand(2).getImm(); 1187 else 1188 return false; 1189 break; 1190 case X86::ADD64i32: 1191 if (!Inst.getOperand(0).isImm()) 1192 return false; 1193 if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX)) 1194 Output = *InputVal + Inst.getOperand(0).getImm(); 1195 else 1196 return false; 1197 break; 1198 1199 case X86::LEA64r: { 1200 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst); 1201 if (!MO) 1202 return false; 1203 1204 if (MO->BaseRegNum == X86::NoRegister || 1205 MO->IndexRegNum != X86::NoRegister || 1206 MO->SegRegNum != X86::NoRegister || MO->DispExpr) 1207 return false; 1208 1209 if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum)) 1210 Output = *InputVal + MO->DispImm; 1211 else 1212 return false; 1213 1214 break; 1215 } 1216 } 1217 return true; 1218 } 1219 1220 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From, 1221 MCPhysReg &To) const override { 1222 switch (Inst.getOpcode()) { 1223 default: 1224 return false; 1225 case X86::LEAVE: 1226 case X86::LEAVE64: 1227 To = getStackPointer(); 1228 From = getFramePointer(); 1229 return true; 1230 case X86::MOV64rr: 1231 To = Inst.getOperand(0).getReg(); 1232 From = Inst.getOperand(1).getReg(); 1233 return true; 1234 } 1235 } 1236 1237 MCPhysReg getStackPointer() const override { return X86::RSP; } 1238 MCPhysReg getFramePointer() const override { return X86::RBP; } 1239 MCPhysReg getFlagsReg() const override { return X86::EFLAGS; } 1240 1241 bool escapesVariable(const MCInst &Inst, 1242 bool HasFramePointer) const override { 1243 int MemOpNo = getMemoryOperandNo(Inst); 1244 const MCInstrDesc &MCII = Info->get(Inst.getOpcode()); 1245 const unsigned NumDefs = MCII.getNumDefs(); 1246 static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |= 1247 getAliases(X86::RBP)); 1248 static BitVector SPAliases(getAliases(X86::RSP)); 1249 1250 // FIXME: PUSH can be technically a leak, but let's ignore this for now 1251 // because a lot of harmless prologue code will spill SP to the stack. 1252 // Unless push is clearly pushing an object address to the stack as 1253 // demonstrated by having a MemOp. 1254 bool IsPush = isPush(Inst); 1255 if (IsPush && MemOpNo == -1) 1256 return false; 1257 1258 // We use this to detect LEA (has memop but does not access mem) 1259 bool AccessMem = MCII.mayLoad() || MCII.mayStore(); 1260 bool DoesLeak = false; 1261 for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) { 1262 // Ignore if SP/BP is used to dereference memory -- that's fine 1263 if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo && 1264 I <= MemOpNo + 5) 1265 continue; 1266 // Ignore if someone is writing to SP/BP 1267 if (I < static_cast<int>(NumDefs)) 1268 continue; 1269 1270 const MCOperand &Operand = Inst.getOperand(I); 1271 if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) { 1272 DoesLeak = true; 1273 break; 1274 } 1275 if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) { 1276 DoesLeak = true; 1277 break; 1278 } 1279 } 1280 1281 // If potential leak, check if it is not just writing to itself/sp/bp 1282 if (DoesLeak) { 1283 DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) { 1284 assert(Operand.isReg()); 1285 MCPhysReg Reg = Operand.getReg(); 1286 return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg]; 1287 }); 1288 } 1289 return DoesLeak; 1290 } 1291 1292 bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override { 1293 unsigned ImmOpNo = -1U; 1294 int MemOpNo = getMemoryOperandNo(Inst); 1295 if (MemOpNo != -1) 1296 ImmOpNo = MemOpNo + X86::AddrDisp; 1297 else 1298 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 1299 ++Index) 1300 if (Inst.getOperand(Index).isImm()) 1301 ImmOpNo = Index; 1302 if (ImmOpNo == -1U) 1303 return false; 1304 1305 MCOperand &Operand = Inst.getOperand(ImmOpNo); 1306 Amt += Operand.getImm(); 1307 Operand.setImm(Amt); 1308 // Check for the need for relaxation 1309 if (int64_t(Amt) == int64_t(int8_t(Amt))) 1310 return true; 1311 1312 // Relax instruction 1313 switch (Inst.getOpcode()) { 1314 case X86::SUB64ri8: 1315 Inst.setOpcode(X86::SUB64ri32); 1316 break; 1317 case X86::ADD64ri8: 1318 Inst.setOpcode(X86::ADD64ri32); 1319 break; 1320 default: 1321 // No need for relaxation 1322 break; 1323 } 1324 return true; 1325 } 1326 1327 /// TODO: this implementation currently works for the most common opcodes that 1328 /// load from memory. It can be extended to work with memory store opcodes as 1329 /// well as more memory load opcodes. 1330 bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData, 1331 uint64_t Offset) const override { 1332 enum CheckSignExt : uint8_t { 1333 NOCHECK = 0, 1334 CHECK8, 1335 CHECK32, 1336 }; 1337 1338 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 1339 struct InstInfo { 1340 // Size in bytes that Inst loads from memory. 1341 uint8_t DataSize; 1342 1343 // True when the target operand has to be duplicated because the opcode 1344 // expects a LHS operand. 1345 bool HasLHS; 1346 1347 // List of checks and corresponding opcodes to be used. We try to use the 1348 // smallest possible immediate value when various sizes are available, 1349 // hence we may need to check whether a larger constant fits in a smaller 1350 // immediate. 1351 CheckList Checks; 1352 }; 1353 1354 InstInfo I; 1355 1356 switch (Inst.getOpcode()) { 1357 default: { 1358 switch (getPopSize(Inst)) { 1359 case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1360 case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1361 case 8: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1362 {NOCHECK, X86::MOV64rm}}}; break; 1363 default: return false; 1364 } 1365 break; 1366 } 1367 1368 // MOV 1369 case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 1370 case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 1371 case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 1372 case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32}, 1373 {NOCHECK, X86::MOV64rm}}}; break; 1374 1375 // MOVZX 1376 case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 1377 case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 1378 case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 1379 1380 // CMP 1381 case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 1382 case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8}, 1383 {NOCHECK, X86::CMP16ri}}}; break; 1384 case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8}, 1385 {NOCHECK, X86::CMP32ri}}}; break; 1386 case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8}, 1387 {CHECK32, X86::CMP64ri32}, 1388 {NOCHECK, X86::CMP64rm}}}; break; 1389 1390 // TEST 1391 case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 1392 case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 1393 case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 1394 case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 1395 {NOCHECK, X86::TEST64mr}}}; break; 1396 1397 // ADD 1398 case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 1399 case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8}, 1400 {NOCHECK, X86::ADD16ri}}}; break; 1401 case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8}, 1402 {NOCHECK, X86::ADD32ri}}}; break; 1403 case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8}, 1404 {CHECK32, X86::ADD64ri32}, 1405 {NOCHECK, X86::ADD64rm}}}; break; 1406 1407 // SUB 1408 case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 1409 case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8}, 1410 {NOCHECK, X86::SUB16ri}}}; break; 1411 case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8}, 1412 {NOCHECK, X86::SUB32ri}}}; break; 1413 case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8}, 1414 {CHECK32, X86::SUB64ri32}, 1415 {NOCHECK, X86::SUB64rm}}}; break; 1416 1417 // AND 1418 case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 1419 case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8}, 1420 {NOCHECK, X86::AND16ri}}}; break; 1421 case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8}, 1422 {NOCHECK, X86::AND32ri}}}; break; 1423 case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8}, 1424 {CHECK32, X86::AND64ri32}, 1425 {NOCHECK, X86::AND64rm}}}; break; 1426 1427 // OR 1428 case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 1429 case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8}, 1430 {NOCHECK, X86::OR16ri}}}; break; 1431 case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8}, 1432 {NOCHECK, X86::OR32ri}}}; break; 1433 case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8}, 1434 {CHECK32, X86::OR64ri32}, 1435 {NOCHECK, X86::OR64rm}}}; break; 1436 1437 // XOR 1438 case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 1439 case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8}, 1440 {NOCHECK, X86::XOR16ri}}}; break; 1441 case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8}, 1442 {NOCHECK, X86::XOR32ri}}}; break; 1443 case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8}, 1444 {CHECK32, X86::XOR64ri32}, 1445 {NOCHECK, X86::XOR64rm}}}; break; 1446 } 1447 1448 // Compute the immediate value. 1449 assert(Offset + I.DataSize <= ConstantData.size() && 1450 "invalid offset for given constant data"); 1451 int64_t ImmVal = 1452 DataExtractor(ConstantData, true, 8).getSigned(&Offset, I.DataSize); 1453 1454 // Compute the new opcode. 1455 unsigned NewOpcode = 0; 1456 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 1457 NewOpcode = Check.second; 1458 if (Check.first == NOCHECK) 1459 break; 1460 if (Check.first == CHECK8 && isInt<8>(ImmVal)) 1461 break; 1462 if (Check.first == CHECK32 && isInt<32>(ImmVal)) 1463 break; 1464 } 1465 if (NewOpcode == Inst.getOpcode()) 1466 return false; 1467 1468 // Modify the instruction. 1469 MCOperand ImmOp = MCOperand::createImm(ImmVal); 1470 uint32_t TargetOpNum = 0; 1471 // Test instruction does not follow the regular pattern of putting the 1472 // memory reference of a load (5 MCOperands) last in the list of operands. 1473 // Since it is not modifying the register operand, it is not treated as 1474 // a destination operand and it is not the first operand as it is in the 1475 // other instructions we treat here. 1476 if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri || 1477 NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32) 1478 TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands; 1479 1480 MCOperand TargetOp = Inst.getOperand(TargetOpNum); 1481 Inst.clear(); 1482 Inst.setOpcode(NewOpcode); 1483 Inst.addOperand(TargetOp); 1484 if (I.HasLHS) 1485 Inst.addOperand(TargetOp); 1486 Inst.addOperand(ImmOp); 1487 1488 return true; 1489 } 1490 1491 /// TODO: this implementation currently works for the most common opcodes that 1492 /// load from memory. It can be extended to work with memory store opcodes as 1493 /// well as more memory load opcodes. 1494 bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override { 1495 unsigned NewOpcode; 1496 1497 switch (Inst.getOpcode()) { 1498 default: { 1499 switch (getPopSize(Inst)) { 1500 case 2: NewOpcode = X86::MOV16rr; break; 1501 case 4: NewOpcode = X86::MOV32rr; break; 1502 case 8: NewOpcode = X86::MOV64rr; break; 1503 default: return false; 1504 } 1505 break; 1506 } 1507 1508 // MOV 1509 case X86::MOV8rm: NewOpcode = X86::MOV8rr; break; 1510 case X86::MOV16rm: NewOpcode = X86::MOV16rr; break; 1511 case X86::MOV32rm: NewOpcode = X86::MOV32rr; break; 1512 case X86::MOV64rm: NewOpcode = X86::MOV64rr; break; 1513 } 1514 1515 // Modify the instruction. 1516 MCOperand RegOp = MCOperand::createReg(RegNum); 1517 MCOperand TargetOp = Inst.getOperand(0); 1518 Inst.clear(); 1519 Inst.setOpcode(NewOpcode); 1520 Inst.addOperand(TargetOp); 1521 Inst.addOperand(RegOp); 1522 1523 return true; 1524 } 1525 1526 bool isRedundantMove(const MCInst &Inst) const override { 1527 switch (Inst.getOpcode()) { 1528 default: 1529 return false; 1530 1531 // MOV 1532 case X86::MOV8rr: 1533 case X86::MOV16rr: 1534 case X86::MOV32rr: 1535 case X86::MOV64rr: 1536 break; 1537 } 1538 1539 assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg()); 1540 return Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg(); 1541 } 1542 1543 bool requiresAlignedAddress(const MCInst &Inst) const override { 1544 const MCInstrDesc &Desc = Info->get(Inst.getOpcode()); 1545 for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) { 1546 const MCOperandInfo &Op = Desc.operands()[I]; 1547 if (Op.OperandType != MCOI::OPERAND_REGISTER) 1548 continue; 1549 if (Op.RegClass == X86::VR128RegClassID) 1550 return true; 1551 } 1552 return false; 1553 } 1554 1555 bool convertJmpToTailCall(MCInst &Inst) override { 1556 if (isTailCall(Inst)) 1557 return false; 1558 1559 int NewOpcode; 1560 switch (Inst.getOpcode()) { 1561 default: 1562 return false; 1563 case X86::JMP_1: 1564 case X86::JMP_2: 1565 case X86::JMP_4: 1566 NewOpcode = X86::JMP_4; 1567 break; 1568 case X86::JMP16m: 1569 case X86::JMP32m: 1570 case X86::JMP64m: 1571 NewOpcode = X86::JMP32m; 1572 break; 1573 case X86::JMP16r: 1574 case X86::JMP32r: 1575 case X86::JMP64r: 1576 NewOpcode = X86::JMP32r; 1577 break; 1578 } 1579 1580 Inst.setOpcode(NewOpcode); 1581 setTailCall(Inst); 1582 return true; 1583 } 1584 1585 bool convertTailCallToJmp(MCInst &Inst) override { 1586 int NewOpcode; 1587 switch (Inst.getOpcode()) { 1588 default: 1589 return false; 1590 case X86::JMP_4: 1591 NewOpcode = X86::JMP_1; 1592 break; 1593 case X86::JMP32m: 1594 NewOpcode = X86::JMP64m; 1595 break; 1596 case X86::JMP32r: 1597 NewOpcode = X86::JMP64r; 1598 break; 1599 } 1600 1601 Inst.setOpcode(NewOpcode); 1602 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1603 clearOffset(Inst); 1604 return true; 1605 } 1606 1607 bool convertTailCallToCall(MCInst &Inst) override { 1608 int NewOpcode; 1609 switch (Inst.getOpcode()) { 1610 default: 1611 return false; 1612 case X86::JMP_4: 1613 NewOpcode = X86::CALL64pcrel32; 1614 break; 1615 case X86::JMP32m: 1616 NewOpcode = X86::CALL64m; 1617 break; 1618 case X86::JMP32r: 1619 NewOpcode = X86::CALL64r; 1620 break; 1621 } 1622 1623 Inst.setOpcode(NewOpcode); 1624 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1625 return true; 1626 } 1627 1628 bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation, 1629 MCContext *Ctx) override { 1630 assert((Inst.getOpcode() == X86::CALL64pcrel32 || 1631 (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) && 1632 "64-bit direct (tail) call instruction expected"); 1633 const auto NewOpcode = 1634 (Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m; 1635 Inst.setOpcode(NewOpcode); 1636 1637 // Replace the first operand and preserve auxiliary operands of 1638 // the instruction. 1639 Inst.erase(Inst.begin()); 1640 Inst.insert(Inst.begin(), 1641 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 1642 Inst.insert(Inst.begin(), 1643 MCOperand::createExpr( // Displacement 1644 MCSymbolRefExpr::create(TargetLocation, 1645 MCSymbolRefExpr::VK_None, *Ctx))); 1646 Inst.insert(Inst.begin(), 1647 MCOperand::createReg(X86::NoRegister)); // IndexReg 1648 Inst.insert(Inst.begin(), 1649 MCOperand::createImm(1)); // ScaleAmt 1650 Inst.insert(Inst.begin(), 1651 MCOperand::createReg(X86::RIP)); // BaseReg 1652 1653 return true; 1654 } 1655 1656 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override { 1657 bool IsTailCall = isTailCall(Inst); 1658 if (IsTailCall) 1659 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1660 if (Inst.getOpcode() == X86::CALL64m || 1661 (Inst.getOpcode() == X86::JMP32m && IsTailCall)) { 1662 Inst.setOpcode(X86::MOV64rm); 1663 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1664 return; 1665 } 1666 if (Inst.getOpcode() == X86::CALL64r || 1667 (Inst.getOpcode() == X86::JMP32r && IsTailCall)) { 1668 Inst.setOpcode(X86::MOV64rr); 1669 Inst.insert(Inst.begin(), MCOperand::createReg(Reg)); 1670 return; 1671 } 1672 LLVM_DEBUG(Inst.dump()); 1673 llvm_unreachable("not implemented"); 1674 } 1675 1676 bool shortenInstruction(MCInst &Inst, 1677 const MCSubtargetInfo &STI) const override { 1678 unsigned OldOpcode = Inst.getOpcode(); 1679 unsigned NewOpcode = OldOpcode; 1680 1681 int MemOpNo = getMemoryOperandNo(Inst); 1682 1683 // Check and remove redundant Address-Size override prefix. 1684 if (opts::X86StripRedundantAddressSize) { 1685 uint64_t TSFlags = Info->get(OldOpcode).TSFlags; 1686 unsigned Flags = Inst.getFlags(); 1687 1688 if (!X86_MC::needsAddressSizeOverride(Inst, STI, MemOpNo, TSFlags) && 1689 Flags & X86::IP_HAS_AD_SIZE) 1690 Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE); 1691 } 1692 1693 // Check and remove EIZ/RIZ. These cases represent ambiguous cases where 1694 // SIB byte is present, but no index is used and modrm alone should have 1695 // been enough. Converting to NoRegister effectively removes the SIB byte. 1696 if (MemOpNo >= 0) { 1697 MCOperand &IndexOp = 1698 Inst.getOperand(static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg); 1699 if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ) 1700 IndexOp = MCOperand::createReg(X86::NoRegister); 1701 } 1702 1703 if (isBranch(Inst)) { 1704 NewOpcode = getShortBranchOpcode(OldOpcode); 1705 } else if (OldOpcode == X86::MOV64ri) { 1706 if (Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1707 const int64_t Imm = 1708 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1709 if (int64_t(Imm) == int64_t(int32_t(Imm))) 1710 NewOpcode = X86::MOV64ri32; 1711 } 1712 } else { 1713 // If it's arithmetic instruction check if signed operand fits in 1 byte. 1714 const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(OldOpcode); 1715 if (ShortOpcode != OldOpcode && 1716 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) { 1717 int64_t Imm = 1718 Inst.getOperand(MCPlus::getNumPrimeOperands(Inst) - 1).getImm(); 1719 if (int64_t(Imm) == int64_t(int8_t(Imm))) 1720 NewOpcode = ShortOpcode; 1721 } 1722 } 1723 1724 if (NewOpcode == OldOpcode) 1725 return false; 1726 1727 Inst.setOpcode(NewOpcode); 1728 return true; 1729 } 1730 1731 bool 1732 convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp, 1733 bool AllowBasePtrStackMemOp) const override { 1734 // - Register-register moves are OK 1735 // - Stores are filtered out by opcode (no store CMOV) 1736 // - Non-stack loads are prohibited (generally unsafe) 1737 // - Stack loads are OK if AllowStackMemOp is true 1738 // - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true 1739 if (isLoad(Inst)) { 1740 // If stack memory operands are not allowed, no loads are allowed 1741 if (!AllowStackMemOp) 1742 return false; 1743 1744 // If stack memory operands are allowed, check if it's a load from stack 1745 bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed; 1746 MCPhysReg Reg; 1747 int32_t SrcImm; 1748 uint16_t StackPtrReg; 1749 int64_t StackOffset; 1750 uint8_t Size; 1751 bool IsStackAccess = 1752 isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm, 1753 StackPtrReg, StackOffset, Size, IsSimple, IsIndexed); 1754 // Prohibit non-stack-based loads 1755 if (!IsStackAccess) 1756 return false; 1757 // If stack memory operands are allowed, check if it's RBP-based 1758 if (!AllowBasePtrStackMemOp && 1759 RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg)) 1760 return false; 1761 } 1762 1763 unsigned NewOpcode = 0; 1764 switch (Inst.getOpcode()) { 1765 case X86::MOV16rr: 1766 NewOpcode = X86::CMOV16rr; 1767 break; 1768 case X86::MOV16rm: 1769 NewOpcode = X86::CMOV16rm; 1770 break; 1771 case X86::MOV32rr: 1772 NewOpcode = X86::CMOV32rr; 1773 break; 1774 case X86::MOV32rm: 1775 NewOpcode = X86::CMOV32rm; 1776 break; 1777 case X86::MOV64rr: 1778 NewOpcode = X86::CMOV64rr; 1779 break; 1780 case X86::MOV64rm: 1781 NewOpcode = X86::CMOV64rm; 1782 break; 1783 default: 1784 return false; 1785 } 1786 Inst.setOpcode(NewOpcode); 1787 // Insert CC at the end of prime operands, before annotations 1788 Inst.insert(Inst.begin() + MCPlus::getNumPrimeOperands(Inst), 1789 MCOperand::createImm(CC)); 1790 // CMOV is a 3-operand MCInst, so duplicate the destination as src1 1791 Inst.insert(Inst.begin(), Inst.getOperand(0)); 1792 return true; 1793 } 1794 1795 bool lowerTailCall(MCInst &Inst) override { 1796 if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) { 1797 Inst.setOpcode(X86::JMP_1); 1798 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall); 1799 return true; 1800 } 1801 return false; 1802 } 1803 1804 const MCSymbol *getTargetSymbol(const MCInst &Inst, 1805 unsigned OpNum = 0) const override { 1806 if (OpNum >= MCPlus::getNumPrimeOperands(Inst)) 1807 return nullptr; 1808 1809 const MCOperand &Op = Inst.getOperand(OpNum); 1810 if (!Op.isExpr()) 1811 return nullptr; 1812 1813 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 1814 if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None) 1815 return nullptr; 1816 1817 return &SymExpr->getSymbol(); 1818 } 1819 1820 // This is the same as the base class, but since we are overriding one of 1821 // getTargetSymbol's signatures above, we need to override all of them. 1822 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override { 1823 return &cast<const MCSymbolRefExpr>(Expr)->getSymbol(); 1824 } 1825 1826 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End, 1827 const MCSymbol *&TBB, const MCSymbol *&FBB, 1828 MCInst *&CondBranch, 1829 MCInst *&UncondBranch) const override { 1830 auto I = End; 1831 1832 // Bottom-up analysis 1833 while (I != Begin) { 1834 --I; 1835 1836 // Ignore nops and CFIs 1837 if (isPseudo(*I)) 1838 continue; 1839 1840 // Stop when we find the first non-terminator 1841 if (!isTerminator(*I)) 1842 break; 1843 1844 if (!isBranch(*I)) 1845 break; 1846 1847 // Handle unconditional branches. 1848 if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 || 1849 I->getOpcode() == X86::JMP_4) && 1850 !isTailCall(*I)) { 1851 // If any code was seen after this unconditional branch, we've seen 1852 // unreachable code. Ignore them. 1853 CondBranch = nullptr; 1854 UncondBranch = &*I; 1855 const MCSymbol *Sym = getTargetSymbol(*I); 1856 assert(Sym != nullptr && 1857 "Couldn't extract BB symbol from jump operand"); 1858 TBB = Sym; 1859 continue; 1860 } 1861 1862 // Handle conditional branches and ignore indirect branches 1863 if (!isUnsupportedBranch(*I) && getCondCode(*I) == X86::COND_INVALID) { 1864 // Indirect branch 1865 return false; 1866 } 1867 1868 if (CondBranch == nullptr) { 1869 const MCSymbol *TargetBB = getTargetSymbol(*I); 1870 if (TargetBB == nullptr) { 1871 // Unrecognized branch target 1872 return false; 1873 } 1874 FBB = TBB; 1875 TBB = TargetBB; 1876 CondBranch = &*I; 1877 continue; 1878 } 1879 1880 llvm_unreachable("multiple conditional branches in one BB"); 1881 } 1882 return true; 1883 } 1884 1885 template <typename Itr> 1886 std::pair<IndirectBranchType, MCInst *> 1887 analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const { 1888 // Analyze PIC-style jump table code template: 1889 // 1890 // lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr 1891 // mov ({%r1|%r2}, %index, 4), {%r2|%r1} 1892 // add %r2, %r1 1893 // jmp *%r1 1894 // 1895 // (with any irrelevant instructions in-between) 1896 // 1897 // When we call this helper we've already determined %r1 and %r2, and 1898 // reverse instruction iterator \p II is pointing to the ADD instruction. 1899 // 1900 // PIC jump table looks like following: 1901 // 1902 // JT: ---------- 1903 // E1:| L1 - JT | 1904 // |----------| 1905 // E2:| L2 - JT | 1906 // |----------| 1907 // | | 1908 // ...... 1909 // En:| Ln - JT | 1910 // ---------- 1911 // 1912 // Where L1, L2, ..., Ln represent labels in the function. 1913 // 1914 // The actual relocations in the table will be of the form: 1915 // 1916 // Ln - JT 1917 // = (Ln - En) + (En - JT) 1918 // = R_X86_64_PC32(Ln) + En - JT 1919 // = R_X86_64_PC32(Ln + offsetof(En)) 1920 // 1921 LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n"); 1922 MCInst *MemLocInstr = nullptr; 1923 const MCInst *MovInstr = nullptr; 1924 while (++II != IE) { 1925 MCInst &Instr = *II; 1926 const MCInstrDesc &InstrDesc = Info->get(Instr.getOpcode()); 1927 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo) && 1928 !InstrDesc.hasDefOfPhysReg(Instr, R2, *RegInfo)) { 1929 // Ignore instructions that don't affect R1, R2 registers. 1930 continue; 1931 } 1932 if (!MovInstr) { 1933 // Expect to see MOV instruction. 1934 if (!isMOVSX64rm32(Instr)) { 1935 LLVM_DEBUG(dbgs() << "MOV instruction expected.\n"); 1936 break; 1937 } 1938 1939 // Check if it's setting %r1 or %r2. In canonical form it sets %r2. 1940 // If it sets %r1 - rename the registers so we have to only check 1941 // a single form. 1942 unsigned MovDestReg = Instr.getOperand(0).getReg(); 1943 if (MovDestReg != R2) 1944 std::swap(R1, R2); 1945 if (MovDestReg != R2) { 1946 LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n"); 1947 break; 1948 } 1949 1950 // Verify operands for MOV. 1951 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1952 if (!MO) 1953 break; 1954 if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 || 1955 MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 || 1956 MO->SegRegNum != X86::NoRegister) 1957 break; 1958 MovInstr = &Instr; 1959 } else { 1960 if (!InstrDesc.hasDefOfPhysReg(Instr, R1, *RegInfo)) 1961 continue; 1962 if (!isLEA64r(Instr)) { 1963 LLVM_DEBUG(dbgs() << "LEA instruction expected\n"); 1964 break; 1965 } 1966 if (Instr.getOperand(0).getReg() != R1) { 1967 LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n"); 1968 break; 1969 } 1970 1971 // Verify operands for LEA. 1972 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr); 1973 if (!MO) 1974 break; 1975 if (MO->BaseRegNum != RegInfo->getProgramCounter() || 1976 MO->IndexRegNum != X86::NoRegister || 1977 MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr) 1978 break; 1979 MemLocInstr = &Instr; 1980 break; 1981 } 1982 } 1983 1984 if (!MemLocInstr) 1985 return std::make_pair(IndirectBranchType::UNKNOWN, nullptr); 1986 1987 LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n"); 1988 return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE, 1989 MemLocInstr); 1990 } 1991 1992 IndirectBranchType analyzeIndirectBranch( 1993 MCInst &Instruction, InstructionIterator Begin, InstructionIterator End, 1994 const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut, 1995 unsigned &IndexRegNumOut, int64_t &DispValueOut, 1996 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override { 1997 // Try to find a (base) memory location from where the address for 1998 // the indirect branch is loaded. For X86-64 the memory will be specified 1999 // in the following format: 2000 // 2001 // {%rip}/{%basereg} + Imm + IndexReg * Scale 2002 // 2003 // We are interested in the cases where Scale == sizeof(uintptr_t) and 2004 // the contents of the memory are presumably an array of pointers to code. 2005 // 2006 // Normal jump table: 2007 // 2008 // jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr 2009 // 2010 // or 2011 // 2012 // mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr 2013 // ... 2014 // jmp %r1 2015 // 2016 // We handle PIC-style jump tables separately. 2017 // 2018 MemLocInstrOut = nullptr; 2019 BaseRegNumOut = X86::NoRegister; 2020 IndexRegNumOut = X86::NoRegister; 2021 DispValueOut = 0; 2022 DispExprOut = nullptr; 2023 2024 std::reverse_iterator<InstructionIterator> II(End); 2025 std::reverse_iterator<InstructionIterator> IE(Begin); 2026 2027 IndirectBranchType Type = IndirectBranchType::UNKNOWN; 2028 2029 // An instruction referencing memory used by jump instruction (directly or 2030 // via register). This location could be an array of function pointers 2031 // in case of indirect tail call, or a jump table. 2032 MCInst *MemLocInstr = nullptr; 2033 2034 if (MCPlus::getNumPrimeOperands(Instruction) == 1) { 2035 // If the indirect jump is on register - try to detect if the 2036 // register value is loaded from a memory location. 2037 assert(Instruction.getOperand(0).isReg() && "register operand expected"); 2038 const unsigned R1 = Instruction.getOperand(0).getReg(); 2039 // Check if one of the previous instructions defines the jump-on register. 2040 for (auto PrevII = II; PrevII != IE; ++PrevII) { 2041 MCInst &PrevInstr = *PrevII; 2042 const MCInstrDesc &PrevInstrDesc = Info->get(PrevInstr.getOpcode()); 2043 2044 if (!PrevInstrDesc.hasDefOfPhysReg(PrevInstr, R1, *RegInfo)) 2045 continue; 2046 2047 if (isMoveMem2Reg(PrevInstr)) { 2048 MemLocInstr = &PrevInstr; 2049 break; 2050 } 2051 if (isADD64rr(PrevInstr)) { 2052 unsigned R2 = PrevInstr.getOperand(2).getReg(); 2053 if (R1 == R2) 2054 return IndirectBranchType::UNKNOWN; 2055 std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2); 2056 break; 2057 } 2058 return IndirectBranchType::UNKNOWN; 2059 } 2060 if (!MemLocInstr) { 2061 // No definition seen for the register in this function so far. Could be 2062 // an input parameter - which means it is an external code reference. 2063 // It also could be that the definition happens to be in the code that 2064 // we haven't processed yet. Since we have to be conservative, return 2065 // as UNKNOWN case. 2066 return IndirectBranchType::UNKNOWN; 2067 } 2068 } else { 2069 MemLocInstr = &Instruction; 2070 } 2071 2072 const MCRegister RIPRegister = RegInfo->getProgramCounter(); 2073 2074 // Analyze the memory location. 2075 std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr); 2076 if (!MO) 2077 return IndirectBranchType::UNKNOWN; 2078 2079 BaseRegNumOut = MO->BaseRegNum; 2080 IndexRegNumOut = MO->IndexRegNum; 2081 DispValueOut = MO->DispImm; 2082 DispExprOut = MO->DispExpr; 2083 2084 if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) || 2085 MO->SegRegNum != X86::NoRegister) 2086 return IndirectBranchType::UNKNOWN; 2087 2088 if (MemLocInstr == &Instruction && 2089 (!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) { 2090 MemLocInstrOut = MemLocInstr; 2091 return IndirectBranchType::POSSIBLE_FIXED_BRANCH; 2092 } 2093 2094 if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2095 (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)) 2096 return IndirectBranchType::UNKNOWN; 2097 2098 if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE && 2099 MO->ScaleImm != PtrSize) 2100 return IndirectBranchType::UNKNOWN; 2101 2102 MemLocInstrOut = MemLocInstr; 2103 2104 return Type; 2105 } 2106 2107 /// Analyze a callsite to see if it could be a virtual method call. This only 2108 /// checks to see if the overall pattern is satisfied, it does not guarantee 2109 /// that the callsite is a true virtual method call. 2110 /// The format of virtual method calls that are recognized is one of the 2111 /// following: 2112 /// 2113 /// Form 1: (found in debug code) 2114 /// add METHOD_OFFSET, %VtableReg 2115 /// mov (%VtableReg), %MethodReg 2116 /// ... 2117 /// call or jmp *%MethodReg 2118 /// 2119 /// Form 2: 2120 /// mov METHOD_OFFSET(%VtableReg), %MethodReg 2121 /// ... 2122 /// call or jmp *%MethodReg 2123 /// 2124 /// Form 3: 2125 /// ... 2126 /// call or jmp *METHOD_OFFSET(%VtableReg) 2127 /// 2128 bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin, 2129 InstructionIterator ForwardEnd, 2130 std::vector<MCInst *> &MethodFetchInsns, 2131 unsigned &VtableRegNum, unsigned &MethodRegNum, 2132 uint64_t &MethodOffset) const override { 2133 VtableRegNum = X86::NoRegister; 2134 MethodRegNum = X86::NoRegister; 2135 MethodOffset = 0; 2136 2137 std::reverse_iterator<InstructionIterator> Itr(ForwardEnd); 2138 std::reverse_iterator<InstructionIterator> End(ForwardBegin); 2139 2140 MCInst &CallInst = *Itr++; 2141 assert(isIndirectBranch(CallInst) || isCall(CallInst)); 2142 2143 // The call can just be jmp offset(reg) 2144 if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) { 2145 if (!MO->DispExpr && MO->BaseRegNum != X86::RIP && 2146 MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) { 2147 MethodRegNum = MO->BaseRegNum; 2148 if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister && 2149 MO->SegRegNum == X86::NoRegister) { 2150 VtableRegNum = MethodRegNum; 2151 MethodOffset = MO->DispImm; 2152 MethodFetchInsns.push_back(&CallInst); 2153 return true; 2154 } 2155 } 2156 return false; 2157 } 2158 if (CallInst.getOperand(0).isReg()) 2159 MethodRegNum = CallInst.getOperand(0).getReg(); 2160 else 2161 return false; 2162 2163 if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) { 2164 VtableRegNum = X86::NoRegister; 2165 MethodRegNum = X86::NoRegister; 2166 return false; 2167 } 2168 2169 // find load from vtable, this may or may not include the method offset 2170 while (Itr != End) { 2171 MCInst &CurInst = *Itr++; 2172 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2173 if (Desc.hasDefOfPhysReg(CurInst, MethodRegNum, *RegInfo)) { 2174 if (!isLoad(CurInst)) 2175 return false; 2176 if (std::optional<X86MemOperand> MO = 2177 evaluateX86MemoryOperand(CurInst)) { 2178 if (!MO->DispExpr && MO->ScaleImm == 1 && 2179 MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP && 2180 MO->BaseRegNum != X86::NoRegister && 2181 MO->IndexRegNum == X86::NoRegister && 2182 MO->SegRegNum == X86::NoRegister && MO->BaseRegNum != X86::RIP) { 2183 VtableRegNum = MO->BaseRegNum; 2184 MethodOffset = MO->DispImm; 2185 MethodFetchInsns.push_back(&CurInst); 2186 if (MethodOffset != 0) 2187 return true; 2188 break; 2189 } 2190 } 2191 return false; 2192 } 2193 } 2194 2195 if (!VtableRegNum) 2196 return false; 2197 2198 // look for any adds affecting the method register. 2199 while (Itr != End) { 2200 MCInst &CurInst = *Itr++; 2201 const MCInstrDesc &Desc = Info->get(CurInst.getOpcode()); 2202 if (Desc.hasDefOfPhysReg(CurInst, VtableRegNum, *RegInfo)) { 2203 if (isADDri(CurInst)) { 2204 assert(!MethodOffset); 2205 MethodOffset = CurInst.getOperand(2).getImm(); 2206 MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst); 2207 break; 2208 } 2209 } 2210 } 2211 2212 return true; 2213 } 2214 2215 bool createStackPointerIncrement(MCInst &Inst, int Size, 2216 bool NoFlagsClobber) const override { 2217 if (NoFlagsClobber) { 2218 Inst.setOpcode(X86::LEA64r); 2219 Inst.clear(); 2220 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2221 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2222 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2223 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2224 Inst.addOperand(MCOperand::createImm(-Size)); // Displacement 2225 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2226 return true; 2227 } 2228 Inst.setOpcode(X86::SUB64ri8); 2229 Inst.clear(); 2230 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2231 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2232 Inst.addOperand(MCOperand::createImm(Size)); 2233 return true; 2234 } 2235 2236 bool createStackPointerDecrement(MCInst &Inst, int Size, 2237 bool NoFlagsClobber) const override { 2238 if (NoFlagsClobber) { 2239 Inst.setOpcode(X86::LEA64r); 2240 Inst.clear(); 2241 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2242 Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg 2243 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2244 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2245 Inst.addOperand(MCOperand::createImm(Size)); // Displacement 2246 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2247 return true; 2248 } 2249 Inst.setOpcode(X86::ADD64ri8); 2250 Inst.clear(); 2251 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2252 Inst.addOperand(MCOperand::createReg(X86::RSP)); 2253 Inst.addOperand(MCOperand::createImm(Size)); 2254 return true; 2255 } 2256 2257 bool createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset, 2258 const MCPhysReg &SrcReg, int Size) const override { 2259 unsigned NewOpcode; 2260 switch (Size) { 2261 default: 2262 return false; 2263 case 2: NewOpcode = X86::MOV16mr; break; 2264 case 4: NewOpcode = X86::MOV32mr; break; 2265 case 8: NewOpcode = X86::MOV64mr; break; 2266 } 2267 Inst.setOpcode(NewOpcode); 2268 Inst.clear(); 2269 Inst.addOperand(MCOperand::createReg(StackReg)); // BaseReg 2270 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2271 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2272 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2273 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2274 Inst.addOperand(MCOperand::createReg(SrcReg)); 2275 return true; 2276 } 2277 2278 bool createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg, 2279 int Offset, const MCPhysReg &DstReg, 2280 int Size) const override { 2281 return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister, 2282 Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister, 2283 DstReg, Size); 2284 } 2285 2286 bool createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale, 2287 const MCPhysReg &IndexReg, int64_t Offset, 2288 const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg, 2289 const MCPhysReg &DstReg, int Size) const override { 2290 unsigned NewOpcode; 2291 switch (Size) { 2292 default: 2293 return false; 2294 case 2: NewOpcode = X86::MOV16rm; break; 2295 case 4: NewOpcode = X86::MOV32rm; break; 2296 case 8: NewOpcode = X86::MOV64rm; break; 2297 } 2298 Inst.setOpcode(NewOpcode); 2299 Inst.clear(); 2300 Inst.addOperand(MCOperand::createReg(DstReg)); 2301 Inst.addOperand(MCOperand::createReg(BaseReg)); 2302 Inst.addOperand(MCOperand::createImm(Scale)); 2303 Inst.addOperand(MCOperand::createReg(IndexReg)); 2304 if (OffsetExpr) 2305 Inst.addOperand(MCOperand::createExpr(OffsetExpr)); // Displacement 2306 else 2307 Inst.addOperand(MCOperand::createImm(Offset)); // Displacement 2308 Inst.addOperand(MCOperand::createReg(AddrSegmentReg)); // AddrSegmentReg 2309 return true; 2310 } 2311 2312 void createLoadImmediate(MCInst &Inst, const MCPhysReg Dest, 2313 uint32_t Imm) const override { 2314 Inst.setOpcode(X86::MOV64ri32); 2315 Inst.clear(); 2316 Inst.addOperand(MCOperand::createReg(Dest)); 2317 Inst.addOperand(MCOperand::createImm(Imm)); 2318 } 2319 2320 bool createIncMemory(MCInst &Inst, const MCSymbol *Target, 2321 MCContext *Ctx) const override { 2322 2323 Inst.setOpcode(X86::LOCK_INC64m); 2324 Inst.clear(); 2325 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 2326 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 2327 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 2328 2329 Inst.addOperand(MCOperand::createExpr( 2330 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, 2331 *Ctx))); // Displacement 2332 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 2333 return true; 2334 } 2335 2336 bool createIJmp32Frag(SmallVectorImpl<MCInst> &Insts, 2337 const MCOperand &BaseReg, const MCOperand &Scale, 2338 const MCOperand &IndexReg, const MCOperand &Offset, 2339 const MCOperand &TmpReg) const override { 2340 // The code fragment we emit here is: 2341 // 2342 // mov32 (%base, %index, scale), %tmpreg 2343 // ijmp *(%tmpreg) 2344 // 2345 MCInst IJmp; 2346 IJmp.setOpcode(X86::JMP64r); 2347 IJmp.addOperand(TmpReg); 2348 2349 MCInst Load; 2350 Load.setOpcode(X86::MOV32rm); 2351 Load.addOperand(TmpReg); 2352 Load.addOperand(BaseReg); 2353 Load.addOperand(Scale); 2354 Load.addOperand(IndexReg); 2355 Load.addOperand(Offset); 2356 Load.addOperand(MCOperand::createReg(X86::NoRegister)); 2357 2358 Insts.push_back(Load); 2359 Insts.push_back(IJmp); 2360 return true; 2361 } 2362 2363 bool createNoop(MCInst &Inst) const override { 2364 Inst.setOpcode(X86::NOOP); 2365 return true; 2366 } 2367 2368 bool createReturn(MCInst &Inst) const override { 2369 Inst.setOpcode(X86::RET64); 2370 return true; 2371 } 2372 2373 InstructionListType createInlineMemcpy(bool ReturnEnd) const override { 2374 InstructionListType Code; 2375 if (ReturnEnd) 2376 Code.emplace_back(MCInstBuilder(X86::LEA64r) 2377 .addReg(X86::RAX) 2378 .addReg(X86::RDI) 2379 .addImm(1) 2380 .addReg(X86::RDX) 2381 .addImm(0) 2382 .addReg(X86::NoRegister)); 2383 else 2384 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2385 .addReg(X86::RAX) 2386 .addReg(X86::RDI)); 2387 2388 Code.emplace_back(MCInstBuilder(X86::MOV32rr) 2389 .addReg(X86::ECX) 2390 .addReg(X86::EDX)); 2391 Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64)); 2392 2393 return Code; 2394 } 2395 2396 InstructionListType createOneByteMemcpy() const override { 2397 InstructionListType Code; 2398 Code.emplace_back(MCInstBuilder(X86::MOV8rm) 2399 .addReg(X86::CL) 2400 .addReg(X86::RSI) 2401 .addImm(0) 2402 .addReg(X86::NoRegister) 2403 .addImm(0) 2404 .addReg(X86::NoRegister)); 2405 Code.emplace_back(MCInstBuilder(X86::MOV8mr) 2406 .addReg(X86::RDI) 2407 .addImm(0) 2408 .addReg(X86::NoRegister) 2409 .addImm(0) 2410 .addReg(X86::NoRegister) 2411 .addReg(X86::CL)); 2412 Code.emplace_back(MCInstBuilder(X86::MOV64rr) 2413 .addReg(X86::RAX) 2414 .addReg(X86::RDI)); 2415 return Code; 2416 } 2417 2418 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm, 2419 const MCSymbol *Target, 2420 MCContext *Ctx) const override { 2421 InstructionListType Code; 2422 Code.emplace_back(MCInstBuilder(X86::CMP64ri8) 2423 .addReg(RegNo) 2424 .addImm(Imm)); 2425 Code.emplace_back(MCInstBuilder(X86::JCC_1) 2426 .addExpr(MCSymbolRefExpr::create( 2427 Target, MCSymbolRefExpr::VK_None, *Ctx)) 2428 .addImm(X86::COND_E)); 2429 return Code; 2430 } 2431 2432 std::optional<Relocation> 2433 createRelocation(const MCFixup &Fixup, 2434 const MCAsmBackend &MAB) const override { 2435 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Fixup.getKind()); 2436 2437 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected"); 2438 const uint64_t RelOffset = Fixup.getOffset(); 2439 2440 uint64_t RelType; 2441 if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) { 2442 switch (FKI.TargetSize) { 2443 default: 2444 return std::nullopt; 2445 case 8: RelType = ELF::R_X86_64_PC8; break; 2446 case 16: RelType = ELF::R_X86_64_PC16; break; 2447 case 32: RelType = ELF::R_X86_64_PC32; break; 2448 case 64: RelType = ELF::R_X86_64_PC64; break; 2449 } 2450 } else { 2451 switch (FKI.TargetSize) { 2452 default: 2453 return std::nullopt; 2454 case 8: RelType = ELF::R_X86_64_8; break; 2455 case 16: RelType = ELF::R_X86_64_16; break; 2456 case 32: RelType = ELF::R_X86_64_32; break; 2457 case 64: RelType = ELF::R_X86_64_64; break; 2458 } 2459 } 2460 2461 auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup); 2462 2463 return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0}); 2464 } 2465 2466 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, 2467 int64_t Addend, MCContext *Ctx, int64_t &Value, 2468 uint64_t RelType) const override { 2469 unsigned ImmOpNo = -1U; 2470 2471 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst); 2472 ++Index) { 2473 if (Inst.getOperand(Index).isImm()) { 2474 ImmOpNo = Index; 2475 // TODO: this is a bit hacky. It finds the correct operand by 2476 // searching for a specific immediate value. If no value is 2477 // provided it defaults to the last immediate operand found. 2478 // This could lead to unexpected results if the instruction 2479 // has more than one immediate with the same value. 2480 if (Inst.getOperand(ImmOpNo).getImm() == Value) 2481 break; 2482 } 2483 } 2484 2485 if (ImmOpNo == -1U) 2486 return false; 2487 2488 Value = Inst.getOperand(ImmOpNo).getImm(); 2489 2490 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType); 2491 2492 return true; 2493 } 2494 2495 bool replaceRegWithImm(MCInst &Inst, unsigned Register, 2496 int64_t Imm) const override { 2497 2498 enum CheckSignExt : uint8_t { 2499 NOCHECK = 0, 2500 CHECK8, 2501 CHECK32, 2502 }; 2503 2504 using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>; 2505 struct InstInfo { 2506 // Size in bytes that Inst loads from memory. 2507 uint8_t DataSize; 2508 2509 // True when the target operand has to be duplicated because the opcode 2510 // expects a LHS operand. 2511 bool HasLHS; 2512 2513 // List of checks and corresponding opcodes to be used. We try to use the 2514 // smallest possible immediate value when various sizes are available, 2515 // hence we may need to check whether a larger constant fits in a smaller 2516 // immediate. 2517 CheckList Checks; 2518 }; 2519 2520 InstInfo I; 2521 2522 switch (Inst.getOpcode()) { 2523 default: { 2524 switch (getPushSize(Inst)) { 2525 2526 case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break; 2527 case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break; 2528 case 8: I = {8, false, {{CHECK8, X86::PUSH64i8}, 2529 {CHECK32, X86::PUSH64i32}, 2530 {NOCHECK, Inst.getOpcode()}}}; break; 2531 default: return false; 2532 } 2533 break; 2534 } 2535 2536 // MOV 2537 case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break; 2538 case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break; 2539 case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break; 2540 case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32}, 2541 {NOCHECK, X86::MOV64ri}}}; break; 2542 2543 case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break; 2544 case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break; 2545 case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break; 2546 case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32}, 2547 {NOCHECK, X86::MOV64mr}}}; break; 2548 2549 // MOVZX 2550 case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break; 2551 case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break; 2552 case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break; 2553 2554 // CMP 2555 case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break; 2556 case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8}, 2557 {NOCHECK, X86::CMP16ri}}}; break; 2558 case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8}, 2559 {NOCHECK, X86::CMP32ri}}}; break; 2560 case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8}, 2561 {CHECK32, X86::CMP64ri32}, 2562 {NOCHECK, X86::CMP64rr}}}; break; 2563 2564 // TEST 2565 case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break; 2566 case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break; 2567 case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break; 2568 case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32}, 2569 {NOCHECK, X86::TEST64rr}}}; break; 2570 2571 // ADD 2572 case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break; 2573 case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8}, 2574 {NOCHECK, X86::ADD16ri}}}; break; 2575 case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8}, 2576 {NOCHECK, X86::ADD32ri}}}; break; 2577 case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8}, 2578 {CHECK32, X86::ADD64ri32}, 2579 {NOCHECK, X86::ADD64rr}}}; break; 2580 2581 // SUB 2582 case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break; 2583 case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8}, 2584 {NOCHECK, X86::SUB16ri}}}; break; 2585 case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8}, 2586 {NOCHECK, X86::SUB32ri}}}; break; 2587 case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8}, 2588 {CHECK32, X86::SUB64ri32}, 2589 {NOCHECK, X86::SUB64rr}}}; break; 2590 2591 // AND 2592 case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break; 2593 case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8}, 2594 {NOCHECK, X86::AND16ri}}}; break; 2595 case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8}, 2596 {NOCHECK, X86::AND32ri}}}; break; 2597 case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8}, 2598 {CHECK32, X86::AND64ri32}, 2599 {NOCHECK, X86::AND64rr}}}; break; 2600 2601 // OR 2602 case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break; 2603 case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8}, 2604 {NOCHECK, X86::OR16ri}}}; break; 2605 case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8}, 2606 {NOCHECK, X86::OR32ri}}}; break; 2607 case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8}, 2608 {CHECK32, X86::OR64ri32}, 2609 {NOCHECK, X86::OR64rr}}}; break; 2610 2611 // XOR 2612 case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break; 2613 case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8}, 2614 {NOCHECK, X86::XOR16ri}}}; break; 2615 case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8}, 2616 {NOCHECK, X86::XOR32ri}}}; break; 2617 case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8}, 2618 {CHECK32, X86::XOR64ri32}, 2619 {NOCHECK, X86::XOR64rr}}}; break; 2620 } 2621 2622 // Compute the new opcode. 2623 unsigned NewOpcode = 0; 2624 for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) { 2625 NewOpcode = Check.second; 2626 if (Check.first == NOCHECK) 2627 break; 2628 if (Check.first == CHECK8 && isInt<8>(Imm)) 2629 break; 2630 if (Check.first == CHECK32 && isInt<32>(Imm)) 2631 break; 2632 } 2633 if (NewOpcode == Inst.getOpcode()) 2634 return false; 2635 2636 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2637 2638 unsigned NumFound = 0; 2639 for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0), 2640 E = InstDesc.getNumOperands(); 2641 Index != E; ++Index) 2642 if (Inst.getOperand(Index).isReg() && 2643 Inst.getOperand(Index).getReg() == Register) 2644 NumFound++; 2645 2646 if (NumFound != 1) 2647 return false; 2648 2649 MCOperand TargetOp = Inst.getOperand(0); 2650 Inst.clear(); 2651 Inst.setOpcode(NewOpcode); 2652 Inst.addOperand(TargetOp); 2653 if (I.HasLHS) 2654 Inst.addOperand(TargetOp); 2655 Inst.addOperand(MCOperand::createImm(Imm)); 2656 2657 return true; 2658 } 2659 2660 bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace, 2661 unsigned ReplaceWith) const override { 2662 2663 // Get the HasLHS value so that iteration can be done 2664 bool HasLHS; 2665 if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) || 2666 X86::isSUB(Inst.getOpcode())) { 2667 HasLHS = true; 2668 } else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) || 2669 X86::isTEST(Inst.getOpcode())) { 2670 HasLHS = false; 2671 } else { 2672 switch (Inst.getOpcode()) { 2673 case X86::MOV8rr: 2674 case X86::MOV8rm: 2675 case X86::MOV8mr: 2676 case X86::MOV8ri: 2677 case X86::MOV16rr: 2678 case X86::MOV16rm: 2679 case X86::MOV16mr: 2680 case X86::MOV16ri: 2681 case X86::MOV32rr: 2682 case X86::MOV32rm: 2683 case X86::MOV32mr: 2684 case X86::MOV32ri: 2685 case X86::MOV64rr: 2686 case X86::MOV64rm: 2687 case X86::MOV64mr: 2688 case X86::MOV64ri: 2689 case X86::MOVZX16rr8: 2690 case X86::MOVZX32rr8: 2691 case X86::MOVZX32rr16: 2692 case X86::MOVSX32rm8: 2693 case X86::MOVSX32rr8: 2694 case X86::MOVSX64rm32: 2695 case X86::LEA64r: 2696 HasLHS = false; 2697 break; 2698 default: 2699 return false; 2700 } 2701 } 2702 2703 const MCInstrDesc &InstDesc = Info->get(Inst.getOpcode()); 2704 2705 bool FoundOne = false; 2706 2707 // Iterate only through src operands that arent also dest operands 2708 for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0), 2709 E = InstDesc.getNumOperands(); 2710 Index != E; ++Index) { 2711 BitVector RegAliases = getAliases(ToReplace, true); 2712 if (!Inst.getOperand(Index).isReg() || 2713 !RegAliases.test(Inst.getOperand(Index).getReg())) 2714 continue; 2715 // Resize register if needed 2716 unsigned SizedReplaceWith = getAliasSized( 2717 ReplaceWith, getRegSize(Inst.getOperand(Index).getReg())); 2718 MCOperand NewOperand = MCOperand::createReg(SizedReplaceWith); 2719 Inst.getOperand(Index) = NewOperand; 2720 FoundOne = true; 2721 } 2722 2723 // Return true if at least one operand was replaced 2724 return FoundOne; 2725 } 2726 2727 bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, 2728 MCContext *Ctx) const override { 2729 Inst.setOpcode(X86::JMP_1); 2730 Inst.addOperand(MCOperand::createExpr( 2731 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx))); 2732 return true; 2733 } 2734 2735 bool createCall(MCInst &Inst, const MCSymbol *Target, 2736 MCContext *Ctx) override { 2737 Inst.setOpcode(X86::CALL64pcrel32); 2738 Inst.addOperand(MCOperand::createExpr( 2739 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2740 return true; 2741 } 2742 2743 bool createTailCall(MCInst &Inst, const MCSymbol *Target, 2744 MCContext *Ctx) override { 2745 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true); 2746 } 2747 2748 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target, 2749 MCContext *Ctx) override { 2750 Seq.clear(); 2751 Seq.emplace_back(); 2752 createDirectCall(Seq.back(), Target, Ctx, /*IsTailCall*/ true); 2753 } 2754 2755 bool createTrap(MCInst &Inst) const override { 2756 Inst.clear(); 2757 Inst.setOpcode(X86::TRAP); 2758 return true; 2759 } 2760 2761 bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, 2762 MCContext *Ctx) const override { 2763 unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); 2764 assert(InvCC != X86::COND_INVALID && "invalid branch instruction"); 2765 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC); 2766 Inst.getOperand(0) = MCOperand::createExpr( 2767 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2768 return true; 2769 } 2770 2771 bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx, 2772 unsigned CC) const override { 2773 if (CC == X86::COND_INVALID) 2774 return false; 2775 Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC); 2776 Inst.getOperand(0) = MCOperand::createExpr( 2777 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2778 return true; 2779 } 2780 2781 unsigned getCanonicalBranchCondCode(unsigned CC) const override { 2782 switch (CC) { 2783 default: return X86::COND_INVALID; 2784 2785 case X86::COND_E: return X86::COND_E; 2786 case X86::COND_NE: return X86::COND_E; 2787 2788 case X86::COND_L: return X86::COND_L; 2789 case X86::COND_GE: return X86::COND_L; 2790 2791 case X86::COND_LE: return X86::COND_G; 2792 case X86::COND_G: return X86::COND_G; 2793 2794 case X86::COND_B: return X86::COND_B; 2795 case X86::COND_AE: return X86::COND_B; 2796 2797 case X86::COND_BE: return X86::COND_A; 2798 case X86::COND_A: return X86::COND_A; 2799 2800 case X86::COND_S: return X86::COND_S; 2801 case X86::COND_NS: return X86::COND_S; 2802 2803 case X86::COND_P: return X86::COND_P; 2804 case X86::COND_NP: return X86::COND_P; 2805 2806 case X86::COND_O: return X86::COND_O; 2807 case X86::COND_NO: return X86::COND_O; 2808 } 2809 } 2810 2811 bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB, 2812 MCContext *Ctx) const override { 2813 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) && 2814 "Invalid instruction"); 2815 Inst.getOperand(0) = MCOperand::createExpr( 2816 MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)); 2817 return true; 2818 } 2819 2820 MCPhysReg getX86R11() const override { return X86::R11; } 2821 2822 unsigned getShortBranchOpcode(unsigned Opcode) const override { 2823 switch (Opcode) { 2824 default: 2825 return Opcode; 2826 case X86::JMP_2: 2827 return X86::JMP_1; 2828 case X86::JMP_4: 2829 return X86::JMP_1; 2830 case X86::JCC_2: 2831 return X86::JCC_1; 2832 case X86::JCC_4: 2833 return X86::JCC_1; 2834 } 2835 } 2836 2837 MCPhysReg getIntArgRegister(unsigned ArgNo) const override { 2838 // FIXME: this should depend on the calling convention. 2839 switch (ArgNo) { 2840 case 0: return X86::RDI; 2841 case 1: return X86::RSI; 2842 case 2: return X86::RDX; 2843 case 3: return X86::RCX; 2844 case 4: return X86::R8; 2845 case 5: return X86::R9; 2846 default: return getNoRegister(); 2847 } 2848 } 2849 2850 void createPause(MCInst &Inst) const override { 2851 Inst.clear(); 2852 Inst.setOpcode(X86::PAUSE); 2853 } 2854 2855 void createLfence(MCInst &Inst) const override { 2856 Inst.clear(); 2857 Inst.setOpcode(X86::LFENCE); 2858 } 2859 2860 bool createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx, 2861 bool IsTailCall) override { 2862 Inst.clear(); 2863 Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 2864 Inst.addOperand(MCOperand::createExpr( 2865 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2866 if (IsTailCall) 2867 setTailCall(Inst); 2868 return true; 2869 } 2870 2871 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target, 2872 MCContext *Ctx, bool IsTailCall) override { 2873 Seq.clear(); 2874 MCInst Inst; 2875 Inst.setOpcode(X86::JMP_1); 2876 Inst.addOperand(MCOperand::createExpr( 2877 MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); 2878 if (IsTailCall) 2879 setTailCall(Inst); 2880 Seq.emplace_back(Inst); 2881 } 2882 2883 bool isConditionalMove(const MCInst &Inst) const override { 2884 unsigned OpCode = Inst.getOpcode(); 2885 return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr || 2886 OpCode == X86::CMOV64rr); 2887 } 2888 2889 bool isBranchOnMem(const MCInst &Inst) const override { 2890 unsigned OpCode = Inst.getOpcode(); 2891 if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) || 2892 OpCode == X86::JMP64m) 2893 return true; 2894 2895 return false; 2896 } 2897 2898 bool isBranchOnReg(const MCInst &Inst) const override { 2899 unsigned OpCode = Inst.getOpcode(); 2900 if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) || 2901 OpCode == X86::JMP64r) 2902 return true; 2903 2904 return false; 2905 } 2906 2907 void createPushRegister(MCInst &Inst, MCPhysReg Reg, 2908 unsigned Size) const override { 2909 Inst.clear(); 2910 unsigned NewOpcode = 0; 2911 if (Reg == X86::EFLAGS) { 2912 switch (Size) { 2913 case 2: NewOpcode = X86::PUSHF16; break; 2914 case 4: NewOpcode = X86::PUSHF32; break; 2915 case 8: NewOpcode = X86::PUSHF64; break; 2916 default: 2917 llvm_unreachable("Unexpected size"); 2918 } 2919 Inst.setOpcode(NewOpcode); 2920 return; 2921 } 2922 switch (Size) { 2923 case 2: NewOpcode = X86::PUSH16r; break; 2924 case 4: NewOpcode = X86::PUSH32r; break; 2925 case 8: NewOpcode = X86::PUSH64r; break; 2926 default: 2927 llvm_unreachable("Unexpected size"); 2928 } 2929 Inst.setOpcode(NewOpcode); 2930 Inst.addOperand(MCOperand::createReg(Reg)); 2931 } 2932 2933 void createPopRegister(MCInst &Inst, MCPhysReg Reg, 2934 unsigned Size) const override { 2935 Inst.clear(); 2936 unsigned NewOpcode = 0; 2937 if (Reg == X86::EFLAGS) { 2938 switch (Size) { 2939 case 2: NewOpcode = X86::POPF16; break; 2940 case 4: NewOpcode = X86::POPF32; break; 2941 case 8: NewOpcode = X86::POPF64; break; 2942 default: 2943 llvm_unreachable("Unexpected size"); 2944 } 2945 Inst.setOpcode(NewOpcode); 2946 return; 2947 } 2948 switch (Size) { 2949 case 2: NewOpcode = X86::POP16r; break; 2950 case 4: NewOpcode = X86::POP32r; break; 2951 case 8: NewOpcode = X86::POP64r; break; 2952 default: 2953 llvm_unreachable("Unexpected size"); 2954 } 2955 Inst.setOpcode(NewOpcode); 2956 Inst.addOperand(MCOperand::createReg(Reg)); 2957 } 2958 2959 void createPushFlags(MCInst &Inst, unsigned Size) const override { 2960 return createPushRegister(Inst, X86::EFLAGS, Size); 2961 } 2962 2963 void createPopFlags(MCInst &Inst, unsigned Size) const override { 2964 return createPopRegister(Inst, X86::EFLAGS, Size); 2965 } 2966 2967 void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value, 2968 unsigned Size) const { 2969 unsigned int Opcode; 2970 switch (Size) { 2971 case 1: Opcode = X86::ADD8ri; break; 2972 case 2: Opcode = X86::ADD16ri; break; 2973 case 4: Opcode = X86::ADD32ri; break; 2974 default: 2975 llvm_unreachable("Unexpected size"); 2976 } 2977 Inst.setOpcode(Opcode); 2978 Inst.clear(); 2979 Inst.addOperand(MCOperand::createReg(Reg)); 2980 Inst.addOperand(MCOperand::createReg(Reg)); 2981 Inst.addOperand(MCOperand::createImm(Value)); 2982 } 2983 2984 void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg, 2985 unsigned Size) const { 2986 unsigned int Opcode; 2987 switch (Size) { 2988 case 1: Opcode = X86::MOV8ri; break; 2989 case 2: Opcode = X86::MOV16ri; break; 2990 case 4: Opcode = X86::MOV32ri; break; 2991 // Writing to a 32-bit register always zeros the upper 32 bits of the 2992 // full-width register 2993 case 8: 2994 Opcode = X86::MOV32ri; 2995 Reg = getAliasSized(Reg, 4); 2996 break; 2997 default: 2998 llvm_unreachable("Unexpected size"); 2999 } 3000 Inst.setOpcode(Opcode); 3001 Inst.clear(); 3002 Inst.addOperand(MCOperand::createReg(Reg)); 3003 Inst.addOperand(MCOperand::createImm(0)); 3004 } 3005 3006 void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const { 3007 Inst.setOpcode(X86::SETCCr); 3008 Inst.clear(); 3009 Inst.addOperand(MCOperand::createReg(Reg)); 3010 Inst.addOperand(MCOperand::createImm(X86::COND_O)); 3011 } 3012 3013 void createX86Lahf(MCInst &Inst) const { 3014 Inst.setOpcode(X86::LAHF); 3015 Inst.clear(); 3016 } 3017 3018 void createX86Sahf(MCInst &Inst) const { 3019 Inst.setOpcode(X86::SAHF); 3020 Inst.clear(); 3021 } 3022 3023 InstructionListType createInstrIncMemory(const MCSymbol *Target, 3024 MCContext *Ctx, 3025 bool IsLeaf) const override { 3026 InstructionListType Instrs(IsLeaf ? 13 : 11); 3027 unsigned int I = 0; 3028 3029 // Don't clobber application red zone (ABI dependent) 3030 if (IsLeaf) 3031 createStackPointerIncrement(Instrs[I++], 128, 3032 /*NoFlagsClobber=*/true); 3033 3034 // Performance improvements based on the optimization discussed at 3035 // https://reviews.llvm.org/D6629 3036 // LAHF/SAHF are used instead of PUSHF/POPF 3037 // PUSHF 3038 createPushRegister(Instrs[I++], X86::RAX, 8); 3039 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3040 createX86Lahf(Instrs[I++]); 3041 createPushRegister(Instrs[I++], X86::RAX, 8); 3042 createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8); 3043 createX86SaveOVFlagToRegister(Instrs[I++], X86::AL); 3044 // LOCK INC 3045 createIncMemory(Instrs[I++], Target, Ctx); 3046 // POPF 3047 createAddRegImm(Instrs[I++], X86::AL, 127, 1); 3048 createPopRegister(Instrs[I++], X86::RAX, 8); 3049 createX86Sahf(Instrs[I++]); 3050 createPopRegister(Instrs[I++], X86::RAX, 8); 3051 3052 if (IsLeaf) 3053 createStackPointerDecrement(Instrs[I], 128, 3054 /*NoFlagsClobber=*/true); 3055 return Instrs; 3056 } 3057 3058 void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg, 3059 int64_t Disp) const { 3060 Inst.setOpcode(X86::XCHG64rm); 3061 Inst.addOperand(MCOperand::createReg(Source)); 3062 Inst.addOperand(MCOperand::createReg(Source)); 3063 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3064 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3065 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3066 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3067 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3068 } 3069 3070 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg, 3071 int64_t Disp) const { 3072 Inst.setOpcode(X86::JMP64m); 3073 Inst.addOperand(MCOperand::createReg(MemBaseReg)); // BaseReg 3074 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3075 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3076 Inst.addOperand(MCOperand::createImm(Disp)); // Displacement 3077 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3078 } 3079 3080 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst, 3081 MCSymbol *HandlerFuncAddr, 3082 int CallSiteID, 3083 MCContext *Ctx) override { 3084 // Check if the target address expression used in the original indirect call 3085 // uses the stack pointer, which we are going to clobber. 3086 static BitVector SPAliases(getAliases(X86::RSP)); 3087 bool UsesSP = any_of(useOperands(CallInst), [&](const MCOperand &Op) { 3088 return Op.isReg() && SPAliases[Op.getReg()]; 3089 }); 3090 3091 InstructionListType Insts; 3092 MCPhysReg TempReg = getIntArgRegister(0); 3093 // Code sequence used to enter indirect call instrumentation helper: 3094 // push %rdi 3095 // add $8, %rsp ;; $rsp may be used in target, so fix it to prev val 3096 // movq target, %rdi ;; via convertIndirectCallTargetToLoad 3097 // sub $8, %rsp ;; restore correct stack value 3098 // push %rdi 3099 // movq $CallSiteID, %rdi 3100 // push %rdi 3101 // callq/jmp HandlerFuncAddr 3102 Insts.emplace_back(); 3103 createPushRegister(Insts.back(), TempReg, 8); 3104 if (UsesSP) { // Only adjust SP if we really need to 3105 Insts.emplace_back(); 3106 createStackPointerDecrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3107 } 3108 Insts.emplace_back(CallInst); 3109 // Insts.back() and CallInst now share the same annotation instruction. 3110 // Strip it from Insts.back(), only preserving tail call annotation. 3111 stripAnnotations(Insts.back(), /*KeepTC=*/true); 3112 convertIndirectCallToLoad(Insts.back(), TempReg); 3113 if (UsesSP) { 3114 Insts.emplace_back(); 3115 createStackPointerIncrement(Insts.back(), 8, /*NoFlagsClobber=*/false); 3116 } 3117 Insts.emplace_back(); 3118 createPushRegister(Insts.back(), TempReg, 8); 3119 Insts.emplace_back(); 3120 createLoadImmediate(Insts.back(), TempReg, CallSiteID); 3121 Insts.emplace_back(); 3122 createPushRegister(Insts.back(), TempReg, 8); 3123 3124 MCInst &NewCallInst = Insts.emplace_back(); 3125 createDirectCall(NewCallInst, HandlerFuncAddr, Ctx, isTailCall(CallInst)); 3126 3127 // Carry over metadata including tail call marker if present. 3128 stripAnnotations(NewCallInst); 3129 moveAnnotations(std::move(CallInst), NewCallInst); 3130 3131 return Insts; 3132 } 3133 3134 InstructionListType createInstrumentedIndCallHandlerExitBB() const override { 3135 const MCPhysReg TempReg = getIntArgRegister(0); 3136 // We just need to undo the sequence created for every ind call in 3137 // instrumentIndirectTarget(), which can be accomplished minimally with: 3138 // popfq 3139 // pop %rdi 3140 // add $16, %rsp 3141 // xchg (%rsp), %rdi 3142 // jmp *-8(%rsp) 3143 InstructionListType Insts(5); 3144 createPopFlags(Insts[0], 8); 3145 createPopRegister(Insts[1], TempReg, 8); 3146 createStackPointerDecrement(Insts[2], 16, /*NoFlagsClobber=*/false); 3147 createSwap(Insts[3], TempReg, X86::RSP, 0); 3148 createIndirectBranch(Insts[4], X86::RSP, -8); 3149 return Insts; 3150 } 3151 3152 InstructionListType 3153 createInstrumentedIndTailCallHandlerExitBB() const override { 3154 const MCPhysReg TempReg = getIntArgRegister(0); 3155 // Same thing as above, but for tail calls 3156 // popfq 3157 // add $16, %rsp 3158 // pop %rdi 3159 // jmp *-16(%rsp) 3160 InstructionListType Insts(4); 3161 createPopFlags(Insts[0], 8); 3162 createStackPointerDecrement(Insts[1], 16, /*NoFlagsClobber=*/false); 3163 createPopRegister(Insts[2], TempReg, 8); 3164 createIndirectBranch(Insts[3], X86::RSP, -16); 3165 return Insts; 3166 } 3167 3168 InstructionListType 3169 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline, 3170 const MCSymbol *IndCallHandler, 3171 MCContext *Ctx) override { 3172 const MCPhysReg TempReg = getIntArgRegister(0); 3173 // Code sequence used to check whether InstrTampoline was initialized 3174 // and call it if so, returns via IndCallHandler. 3175 // pushfq 3176 // mov InstrTrampoline,%rdi 3177 // cmp $0x0,%rdi 3178 // je IndCallHandler 3179 // callq *%rdi 3180 // jmpq IndCallHandler 3181 InstructionListType Insts; 3182 Insts.emplace_back(); 3183 createPushFlags(Insts.back(), 8); 3184 Insts.emplace_back(); 3185 createMove(Insts.back(), InstrTrampoline, TempReg, Ctx); 3186 InstructionListType cmpJmp = createCmpJE(TempReg, 0, IndCallHandler, Ctx); 3187 Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end()); 3188 Insts.emplace_back(); 3189 Insts.back().setOpcode(X86::CALL64r); 3190 Insts.back().addOperand(MCOperand::createReg(TempReg)); 3191 Insts.emplace_back(); 3192 createDirectCall(Insts.back(), IndCallHandler, Ctx, /*IsTailCall*/ true); 3193 return Insts; 3194 } 3195 3196 InstructionListType createNumCountersGetter(MCContext *Ctx) const override { 3197 InstructionListType Insts(2); 3198 MCSymbol *NumLocs = Ctx->getOrCreateSymbol("__bolt_num_counters"); 3199 createMove(Insts[0], NumLocs, X86::EAX, Ctx); 3200 createReturn(Insts[1]); 3201 return Insts; 3202 } 3203 3204 InstructionListType 3205 createInstrLocationsGetter(MCContext *Ctx) const override { 3206 InstructionListType Insts(2); 3207 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_locations"); 3208 createLea(Insts[0], Locs, X86::EAX, Ctx); 3209 createReturn(Insts[1]); 3210 return Insts; 3211 } 3212 3213 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override { 3214 InstructionListType Insts(2); 3215 MCSymbol *Locs = Ctx->getOrCreateSymbol("__bolt_instr_tables"); 3216 createLea(Insts[0], Locs, X86::EAX, Ctx); 3217 createReturn(Insts[1]); 3218 return Insts; 3219 } 3220 3221 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override { 3222 InstructionListType Insts(2); 3223 MCSymbol *NumFuncs = Ctx->getOrCreateSymbol("__bolt_instr_num_funcs"); 3224 createMove(Insts[0], NumFuncs, X86::EAX, Ctx); 3225 createReturn(Insts[1]); 3226 return Insts; 3227 } 3228 3229 InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym, 3230 MCContext *Ctx) const override { 3231 InstructionListType Insts(1); 3232 createUncondBranch(Insts[0], TgtSym, Ctx); 3233 return Insts; 3234 } 3235 3236 InstructionListType createDummyReturnFunction(MCContext *Ctx) const override { 3237 InstructionListType Insts(1); 3238 createReturn(Insts[0]); 3239 return Insts; 3240 } 3241 3242 BlocksVectorTy indirectCallPromotion( 3243 const MCInst &CallInst, 3244 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3245 const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms, 3246 const std::vector<MCInst *> &MethodFetchInsns, 3247 const bool MinimizeCodeSize, MCContext *Ctx) override { 3248 const bool IsTailCall = isTailCall(CallInst); 3249 const bool IsJumpTable = getJumpTable(CallInst) != 0; 3250 BlocksVectorTy Results; 3251 3252 // Label for the current code block. 3253 MCSymbol *NextTarget = nullptr; 3254 3255 // The join block which contains all the instructions following CallInst. 3256 // MergeBlock remains null if CallInst is a tail call. 3257 MCSymbol *MergeBlock = nullptr; 3258 3259 unsigned FuncAddrReg = X86::R10; 3260 3261 const bool LoadElim = !VtableSyms.empty(); 3262 assert((!LoadElim || VtableSyms.size() == Targets.size()) && 3263 "There must be a vtable entry for every method " 3264 "in the targets vector."); 3265 3266 if (MinimizeCodeSize && !LoadElim) { 3267 std::set<unsigned> UsedRegs; 3268 3269 for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(CallInst); ++I) { 3270 const MCOperand &Op = CallInst.getOperand(I); 3271 if (Op.isReg()) 3272 UsedRegs.insert(Op.getReg()); 3273 } 3274 3275 if (UsedRegs.count(X86::R10) == 0) 3276 FuncAddrReg = X86::R10; 3277 else if (UsedRegs.count(X86::R11) == 0) 3278 FuncAddrReg = X86::R11; 3279 else 3280 return Results; 3281 } 3282 3283 const auto jumpToMergeBlock = [&](InstructionListType &NewCall) { 3284 assert(MergeBlock); 3285 NewCall.push_back(CallInst); 3286 MCInst &Merge = NewCall.back(); 3287 Merge.clear(); 3288 createUncondBranch(Merge, MergeBlock, Ctx); 3289 }; 3290 3291 for (unsigned int i = 0; i < Targets.size(); ++i) { 3292 Results.emplace_back(NextTarget, InstructionListType()); 3293 InstructionListType *NewCall = &Results.back().second; 3294 3295 if (MinimizeCodeSize && !LoadElim) { 3296 // Load the call target into FuncAddrReg. 3297 NewCall->push_back(CallInst); // Copy CallInst in order to get SMLoc 3298 MCInst &Target = NewCall->back(); 3299 Target.clear(); 3300 Target.setOpcode(X86::MOV64ri32); 3301 Target.addOperand(MCOperand::createReg(FuncAddrReg)); 3302 if (Targets[i].first) { 3303 // Is this OK? 3304 Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3305 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3306 } else { 3307 const uint64_t Addr = Targets[i].second; 3308 // Immediate address is out of sign extended 32 bit range. 3309 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3310 return BlocksVectorTy(); 3311 3312 Target.addOperand(MCOperand::createImm(Addr)); 3313 } 3314 3315 // Compare current call target to a specific address. 3316 NewCall->push_back(CallInst); 3317 MCInst &Compare = NewCall->back(); 3318 Compare.clear(); 3319 if (isBranchOnReg(CallInst)) 3320 Compare.setOpcode(X86::CMP64rr); 3321 else if (CallInst.getOpcode() == X86::CALL64pcrel32) 3322 Compare.setOpcode(X86::CMP64ri32); 3323 else 3324 Compare.setOpcode(X86::CMP64rm); 3325 3326 Compare.addOperand(MCOperand::createReg(FuncAddrReg)); 3327 3328 // TODO: Would be preferable to only load this value once. 3329 for (unsigned i = 0; 3330 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3331 if (!CallInst.getOperand(i).isInst()) 3332 Compare.addOperand(CallInst.getOperand(i)); 3333 } else { 3334 // Compare current call target to a specific address. 3335 NewCall->push_back(CallInst); 3336 MCInst &Compare = NewCall->back(); 3337 Compare.clear(); 3338 if (isBranchOnReg(CallInst)) 3339 Compare.setOpcode(X86::CMP64ri32); 3340 else 3341 Compare.setOpcode(X86::CMP64mi32); 3342 3343 // Original call address. 3344 for (unsigned i = 0; 3345 i < Info->get(CallInst.getOpcode()).getNumOperands(); ++i) 3346 if (!CallInst.getOperand(i).isInst()) 3347 Compare.addOperand(CallInst.getOperand(i)); 3348 3349 // Target address. 3350 if (Targets[i].first || LoadElim) { 3351 const MCSymbol *Sym = 3352 LoadElim ? VtableSyms[i].first : Targets[i].first; 3353 const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0; 3354 const MCExpr *Expr = MCSymbolRefExpr::create(Sym, *Ctx); 3355 if (Addend) 3356 Expr = MCBinaryExpr::createAdd( 3357 Expr, MCConstantExpr::create(Addend, *Ctx), *Ctx); 3358 Compare.addOperand(MCOperand::createExpr(Expr)); 3359 } else { 3360 const uint64_t Addr = Targets[i].second; 3361 // Immediate address is out of sign extended 32 bit range. 3362 if (int64_t(Addr) != int64_t(int32_t(Addr))) 3363 return BlocksVectorTy(); 3364 3365 Compare.addOperand(MCOperand::createImm(Addr)); 3366 } 3367 } 3368 3369 // jump to next target compare. 3370 NextTarget = 3371 Ctx->createNamedTempSymbol(); // generate label for the next block 3372 NewCall->push_back(CallInst); 3373 3374 if (IsJumpTable) { 3375 MCInst &Je = NewCall->back(); 3376 3377 // Jump to next compare if target addresses don't match. 3378 Je.clear(); 3379 Je.setOpcode(X86::JCC_1); 3380 if (Targets[i].first) 3381 Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3382 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3383 else 3384 Je.addOperand(MCOperand::createImm(Targets[i].second)); 3385 3386 Je.addOperand(MCOperand::createImm(X86::COND_E)); 3387 assert(!isInvoke(CallInst)); 3388 } else { 3389 MCInst &Jne = NewCall->back(); 3390 3391 // Jump to next compare if target addresses don't match. 3392 Jne.clear(); 3393 Jne.setOpcode(X86::JCC_1); 3394 Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3395 NextTarget, MCSymbolRefExpr::VK_None, *Ctx))); 3396 Jne.addOperand(MCOperand::createImm(X86::COND_NE)); 3397 3398 // Call specific target directly. 3399 Results.emplace_back(Ctx->createNamedTempSymbol(), 3400 InstructionListType()); 3401 NewCall = &Results.back().second; 3402 NewCall->push_back(CallInst); 3403 MCInst &CallOrJmp = NewCall->back(); 3404 3405 CallOrJmp.clear(); 3406 3407 if (MinimizeCodeSize && !LoadElim) { 3408 CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r); 3409 CallOrJmp.addOperand(MCOperand::createReg(FuncAddrReg)); 3410 } else { 3411 CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32); 3412 3413 if (Targets[i].first) 3414 CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3415 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3416 else 3417 CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second)); 3418 } 3419 if (IsTailCall) 3420 setTailCall(CallOrJmp); 3421 3422 if (CallOrJmp.getOpcode() == X86::CALL64r || 3423 CallOrJmp.getOpcode() == X86::CALL64pcrel32) { 3424 if (std::optional<uint32_t> Offset = getOffset(CallInst)) 3425 // Annotated as duplicated call 3426 setOffset(CallOrJmp, *Offset); 3427 } 3428 3429 if (isInvoke(CallInst) && !isInvoke(CallOrJmp)) { 3430 // Copy over any EH or GNU args size information from the original 3431 // call. 3432 std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst); 3433 if (EHInfo) 3434 addEHInfo(CallOrJmp, *EHInfo); 3435 int64_t GnuArgsSize = getGnuArgsSize(CallInst); 3436 if (GnuArgsSize >= 0) 3437 addGnuArgsSize(CallOrJmp, GnuArgsSize); 3438 } 3439 3440 if (!IsTailCall) { 3441 // The fallthrough block for the most common target should be 3442 // the merge block. 3443 if (i == 0) { 3444 // Fallthrough to merge block. 3445 MergeBlock = Ctx->createNamedTempSymbol(); 3446 } else { 3447 // Insert jump to the merge block if we are not doing a fallthrough. 3448 jumpToMergeBlock(*NewCall); 3449 } 3450 } 3451 } 3452 } 3453 3454 // Cold call block. 3455 Results.emplace_back(NextTarget, InstructionListType()); 3456 InstructionListType &NewCall = Results.back().second; 3457 for (const MCInst *Inst : MethodFetchInsns) 3458 if (Inst != &CallInst) 3459 NewCall.push_back(*Inst); 3460 NewCall.push_back(CallInst); 3461 3462 // Jump to merge block from cold call block 3463 if (!IsTailCall && !IsJumpTable) { 3464 jumpToMergeBlock(NewCall); 3465 3466 // Record merge block 3467 Results.emplace_back(MergeBlock, InstructionListType()); 3468 } 3469 3470 return Results; 3471 } 3472 3473 BlocksVectorTy jumpTablePromotion( 3474 const MCInst &IJmpInst, 3475 const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets, 3476 const std::vector<MCInst *> &TargetFetchInsns, 3477 MCContext *Ctx) const override { 3478 assert(getJumpTable(IJmpInst) != 0); 3479 uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg"); 3480 if (IndexReg == 0) 3481 return BlocksVectorTy(); 3482 3483 BlocksVectorTy Results; 3484 3485 // Label for the current code block. 3486 MCSymbol *NextTarget = nullptr; 3487 3488 for (unsigned int i = 0; i < Targets.size(); ++i) { 3489 Results.emplace_back(NextTarget, InstructionListType()); 3490 InstructionListType *CurBB = &Results.back().second; 3491 3492 // Compare current index to a specific index. 3493 CurBB->emplace_back(MCInst()); 3494 MCInst &CompareInst = CurBB->back(); 3495 CompareInst.setLoc(IJmpInst.getLoc()); 3496 CompareInst.setOpcode(X86::CMP64ri32); 3497 CompareInst.addOperand(MCOperand::createReg(IndexReg)); 3498 3499 const uint64_t CaseIdx = Targets[i].second; 3500 // Immediate address is out of sign extended 32 bit range. 3501 if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx))) 3502 return BlocksVectorTy(); 3503 3504 CompareInst.addOperand(MCOperand::createImm(CaseIdx)); 3505 shortenInstruction(CompareInst, *Ctx->getSubtargetInfo()); 3506 3507 // jump to next target compare. 3508 NextTarget = 3509 Ctx->createNamedTempSymbol(); // generate label for the next block 3510 CurBB->push_back(MCInst()); 3511 3512 MCInst &JEInst = CurBB->back(); 3513 JEInst.setLoc(IJmpInst.getLoc()); 3514 3515 // Jump to target if indices match 3516 JEInst.setOpcode(X86::JCC_1); 3517 JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create( 3518 Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx))); 3519 JEInst.addOperand(MCOperand::createImm(X86::COND_E)); 3520 } 3521 3522 // Cold call block. 3523 Results.emplace_back(NextTarget, InstructionListType()); 3524 InstructionListType &CurBB = Results.back().second; 3525 for (const MCInst *Inst : TargetFetchInsns) 3526 if (Inst != &IJmpInst) 3527 CurBB.push_back(*Inst); 3528 3529 CurBB.push_back(IJmpInst); 3530 3531 return Results; 3532 } 3533 3534 private: 3535 bool createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3536 MCContext *Ctx) const { 3537 Inst.setOpcode(X86::MOV64rm); 3538 Inst.addOperand(MCOperand::createReg(Reg)); 3539 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3540 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3541 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3542 Inst.addOperand(MCOperand::createExpr( 3543 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3544 *Ctx))); // Displacement 3545 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3546 3547 return true; 3548 } 3549 3550 bool createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg, 3551 MCContext *Ctx) const { 3552 Inst.setOpcode(X86::LEA64r); 3553 Inst.addOperand(MCOperand::createReg(Reg)); 3554 Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg 3555 Inst.addOperand(MCOperand::createImm(1)); // ScaleAmt 3556 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg 3557 Inst.addOperand(MCOperand::createExpr( 3558 MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None, 3559 *Ctx))); // Displacement 3560 Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg 3561 return true; 3562 } 3563 }; 3564 3565 } // namespace 3566 3567 namespace llvm { 3568 namespace bolt { 3569 3570 MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis, 3571 const MCInstrInfo *Info, 3572 const MCRegisterInfo *RegInfo) { 3573 return new X86MCPlusBuilder(Analysis, Info, RegInfo); 3574 } 3575 3576 } // namespace bolt 3577 } // namespace llvm 3578